From 6bbf8922e4cff20d70d6e9fa5e9ffefa28c6f75c Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 6 Sep 2024 19:21:51 -0400 Subject: [PATCH] feat(forge/cli): adds support for running targets against multiple platforms --- blueprint/schema/_embed/schema.cue | 4 + blueprint/schema/schema.go | 4 + blueprint/schema/schema_go_gen.cue | 4 + forge/cli/cmd/cmds/run.go | 12 +-- forge/cli/cmd/cmds/util.go | 10 ++- forge/cli/cmd/testdata/run/1.txt | 6 +- forge/cli/cmd/testdata/run/2.txt | 6 +- forge/cli/cmd/testdata/run/3.txt | 14 +++- forge/cli/cmd/testdata/run/4.txt | 16 +++- forge/cli/cmd/testdata/run/5.txt | 6 +- forge/cli/pkg/earthly/earthly.go | 54 +++++++++---- forge/cli/pkg/earthly/earthly_test.go | 107 +++++++++++++++++--------- forge/cli/pkg/earthly/options.go | 8 +- 13 files changed, 177 insertions(+), 74 deletions(-) diff --git a/blueprint/schema/_embed/schema.cue b/blueprint/schema/_embed/schema.cue index 0703005..4e23f0b 100644 --- a/blueprint/schema/_embed/schema.cue +++ b/blueprint/schema/_embed/schema.cue @@ -160,6 +160,10 @@ version: "1.0" [string]: string } @go(Args,map[string]string) + // Platforms contains the platforms to run the target against. + // +optional + platforms?: [...string] @go(Platforms,[]string) + // Privileged determines if the target should run in privileged mode. // +optional privileged?: null | bool @go(Privileged,*bool) diff --git a/blueprint/schema/schema.go b/blueprint/schema/schema.go index e18afaf..730941d 100644 --- a/blueprint/schema/schema.go +++ b/blueprint/schema/schema.go @@ -161,6 +161,10 @@ type Target struct { // +optional Args map[string]string `json:"args"` + // Platforms contains the platforms to run the target against. + // +optional + Platforms []string `json:"platforms"` + // Privileged determines if the target should run in privileged mode. // +optional Privileged *bool `json:"privileged"` diff --git a/blueprint/schema/schema_go_gen.cue b/blueprint/schema/schema_go_gen.cue index 7de5cce..b5a8f74 100644 --- a/blueprint/schema/schema_go_gen.cue +++ b/blueprint/schema/schema_go_gen.cue @@ -155,6 +155,10 @@ package schema // +optional args?: {[string]: string} @go(Args,map[string]string) + // Platforms contains the platforms to run the target against. + // +optional + platforms?: [...string] @go(Platforms,[]string) + // Privileged determines if the target should run in privileged mode. // +optional privileged?: null | bool @go(Privileged,*bool) diff --git a/forge/cli/cmd/cmds/run.go b/forge/cli/cmd/cmds/run.go index 4616b1a..b083d41 100644 --- a/forge/cli/cmd/cmds/run.go +++ b/forge/cli/cmd/cmds/run.go @@ -11,12 +11,12 @@ import ( ) type RunCmd struct { - Artifact string `short:"a" help:"Dump all produced artifacts to the given path."` - CI bool `help:"Run the target in CI mode."` - Local bool `short:"l" help:"Forces the target to run locally (ignores satellite)."` - Path string `arg:"" help:"The path to the target to execute (i.e., ./dir1+test)."` - Platform string `short:"p" help:"Run the target with the given platform."` - Pretty bool `help:"Pretty print JSON output."` + Artifact string `short:"a" help:"Dump all produced artifacts to the given path."` + CI bool `help:"Run the target in CI mode."` + Local bool `short:"l" help:"Forces the target to run locally (ignores satellite)."` + Path string `arg:"" help:"The path to the target to execute (i.e., ./dir1+test)."` + Platform []string `short:"p" help:"Run the target with the given platform."` + Pretty bool `help:"Pretty print JSON output."` } func (c *RunCmd) Run(logger *slog.Logger) error { diff --git a/forge/cli/cmd/cmds/util.go b/forge/cli/cmd/cmds/util.go index b894a12..9f4acd7 100644 --- a/forge/cli/cmd/cmds/util.go +++ b/forge/cli/cmd/cmds/util.go @@ -39,6 +39,11 @@ func generateOpts(target string, flags *RunCmd, config *schema.Blueprint) []eart opts = append(opts, earthly.WithTargetArgs(args...)) } + // We only run multiple platforms in CI mode to avoid issues with local builds. + if targetConfig.Platforms != nil && flags.CI { + opts = append(opts, earthly.WithPlatforms(targetConfig.Platforms...)) + } + if targetConfig.Privileged != nil && *targetConfig.Privileged { opts = append(opts, earthly.WithPrivileged()) } @@ -66,8 +71,9 @@ func generateOpts(target string, flags *RunCmd, config *schema.Blueprint) []eart opts = append(opts, earthly.WithCI()) } - if flags.Platform != "" { - opts = append(opts, earthly.WithPlatform(flags.Platform)) + // Users can explicitly set the platforms to use without being in CI mode. + if flags.Platform != nil { + opts = append(opts, earthly.WithPlatforms(flags.Platform...)) } } diff --git a/forge/cli/cmd/testdata/run/1.txt b/forge/cli/cmd/testdata/run/1.txt index c2dced7..ee5b772 100644 --- a/forge/cli/cmd/testdata/run/1.txt +++ b/forge/cli/cmd/testdata/run/1.txt @@ -1,12 +1,14 @@ -exec forge run ./dir1+test +exec forge run --platform test ./dir1+test cmp stdout golden.txt -- golden.txt -- earthly +--platform +test ./dir1+test Image ./dir1+test output as test -{"artifacts":{},"images":{"./dir1+test":"test"}} +{"test":{"artifacts":{},"images":{"./dir1+test":"test"}}} -- earthly_stdout.txt -- Image ./dir1+test output as test -- dir1/Earthfile -- diff --git a/forge/cli/cmd/testdata/run/2.txt b/forge/cli/cmd/testdata/run/2.txt index ae267cd..d05bc66 100644 --- a/forge/cli/cmd/testdata/run/2.txt +++ b/forge/cli/cmd/testdata/run/2.txt @@ -1,15 +1,17 @@ -exec forge run --artifact output ./dir1+test +exec forge run --platform test --artifact output ./dir1+test cmp stdout golden.txt -- golden.txt -- earthly +--platform +test --artifact ./dir1+test/* output/ Image ./dir1+test output as test Artifact ./dir1+test output as test -{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}} +{"test":{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}}} -- earthly_stdout.txt -- Image ./dir1+test output as test Artifact ./dir1+test output as test diff --git a/forge/cli/cmd/testdata/run/3.txt b/forge/cli/cmd/testdata/run/3.txt index 22824f0..5f1ad5f 100644 --- a/forge/cli/cmd/testdata/run/3.txt +++ b/forge/cli/cmd/testdata/run/3.txt @@ -1,16 +1,24 @@ -exec forge run --ci --platform test ./dir1+test +exec forge run --ci --platform test --platform test1 ./dir1+test cmp stdout golden.txt -- golden.txt -- earthly ---ci --platform test +--ci +./dir1+test +Image ./dir1+test output as test +Artifact ./dir1+test output as test + +earthly +--platform +test1 +--ci ./dir1+test Image ./dir1+test output as test Artifact ./dir1+test output as test -{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}} +{"test":{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}},"test1":{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}}} -- earthly_stdout.txt -- Image ./dir1+test output as test Artifact ./dir1+test output as test diff --git a/forge/cli/cmd/testdata/run/4.txt b/forge/cli/cmd/testdata/run/4.txt index 4c965a0..bc66b7c 100644 --- a/forge/cli/cmd/testdata/run/4.txt +++ b/forge/cli/cmd/testdata/run/4.txt @@ -1,13 +1,24 @@ -exec forge run ./dir1+test +exec forge run --ci ./dir1+test cmp stdout golden.txt -- golden.txt -- earthly +--platform +test --allow-privileged +--ci ./dir1+test Image ./dir1+test output as test -{"artifacts":{},"images":{"./dir1+test":"test"}} +earthly +--platform +test1 +--allow-privileged +--ci +./dir1+test +Image ./dir1+test output as test + +{"test":{"artifacts":{},"images":{"./dir1+test":"test"}},"test1":{"artifacts":{},"images":{"./dir1+test":"test"}}} -- earthly_stdout.txt -- Image ./dir1+test output as test -- dir1/blueprint.cue -- @@ -17,6 +28,7 @@ project: { ci: { targets: { test: { + platforms: ["test", "test1"] privileged: true } } diff --git a/forge/cli/cmd/testdata/run/5.txt b/forge/cli/cmd/testdata/run/5.txt index 1ceb257..c234d43 100644 --- a/forge/cli/cmd/testdata/run/5.txt +++ b/forge/cli/cmd/testdata/run/5.txt @@ -1,16 +1,18 @@ -exec forge run ./dir1+test +exec forge run --platform test ./dir1+test cmp stdout golden.txt -- dir1/Secretfile -- {"secret_key": "secret_value"} -- golden.txt -- earthly +--platform +test --allow-privileged ./dir1+test EARTHLY_SECRETS=secret_id=secret_value Image ./dir1+test output as test -{"artifacts":{},"images":{"./dir1+test":"test"}} +{"test":{"artifacts":{},"images":{"./dir1+test":"test"}}} -- earthly_stdout.txt -- Image ./dir1+test output as test -- dir1/blueprint.cue -- diff --git a/forge/cli/pkg/earthly/earthly.go b/forge/cli/pkg/earthly/earthly.go index 5bc0de6..bf4aad3 100644 --- a/forge/cli/pkg/earthly/earthly.go +++ b/forge/cli/pkg/earthly/earthly.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "regexp" + "runtime" "strconv" "strings" @@ -26,9 +27,10 @@ type EarthlySecret struct { // earthlyExecutorOptions contains the configuration options for an // EarthlyExecutor. type earthlyExecutorOptions struct { - artifact string - ci bool - retries int + artifact string + ci bool + platforms []string + retries int } // EarthlyExecutor is an Executor that runs Earthly targets. @@ -52,7 +54,7 @@ type EarthlyExecutionResult struct { // Run executes the Earthly target and returns the resulting images and // artifacts. -func (e EarthlyExecutor) Run() (EarthlyExecutionResult, error) { +func (e EarthlyExecutor) Run() (map[string]EarthlyExecutionResult, error) { var ( err error secrets []EarthlySecret @@ -61,7 +63,7 @@ func (e EarthlyExecutor) Run() (EarthlyExecutionResult, error) { if e.secrets != nil { secrets, err = e.buildSecrets() if err != nil { - return EarthlyExecutionResult{}, err + return nil, err } var secretString []string @@ -75,30 +77,45 @@ func (e EarthlyExecutor) Run() (EarthlyExecutionResult, error) { } } - var output []byte - arguments := e.buildArguments() - for i := 0; i < e.opts.retries+1; i++ { - e.logger.Info("Executing Earthly", "attempt", i, "retries", e.opts.retries, "arguments", arguments) - output, err = e.executor.Execute("earthly", arguments) - if err == nil { - break + if e.opts.platforms == nil { + e.opts.platforms = []string{getNativePlatform()} + } + + results := make(map[string]EarthlyExecutionResult) + for _, platform := range e.opts.platforms { + var output []byte + + for i := 0; i < e.opts.retries+1; i++ { + arguments := e.buildArguments(platform) + + e.logger.Info("Executing Earthly", "attempt", i, "retries", e.opts.retries, "arguments", arguments, "platform", platform) + output, err = e.executor.Execute("earthly", arguments) + if err == nil { + break + } + + e.logger.Error("Failed to run Earthly", "error", err) } - e.logger.Error("Failed to run Earthly", "error", err) + results[platform] = parseResult(string(output)) } if err != nil { e.logger.Error("Failed to run Earthly", "error", err) - return EarthlyExecutionResult{}, fmt.Errorf("failed to run Earthly: %w", err) + return nil, fmt.Errorf("failed to run Earthly: %w", err) } - return parseResult(string(output)), nil + return results, nil } // buildArguments constructs the arguments to pass to the Earthly target. -func (e *EarthlyExecutor) buildArguments() []string { +func (e *EarthlyExecutor) buildArguments(platform string) []string { var earthlyArgs []string + if platform != getNativePlatform() { + earthlyArgs = append(earthlyArgs, "--platform", platform) + } + earthlyArgs = append(earthlyArgs, e.earthlyArgs...) if e.opts.artifact != "" { @@ -186,6 +203,11 @@ func NewEarthlyExecutor( return e } +// getNativePlatform returns the native platform of the current machine. +func getNativePlatform() string { + return fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) +} + // parseResult parses the output of an Earthly execution and returns the // resulting images and artifacts. func parseResult(output string) EarthlyExecutionResult { diff --git a/forge/cli/pkg/earthly/earthly_test.go b/forge/cli/pkg/earthly/earthly_test.go index ab692d0..9de33b2 100644 --- a/forge/cli/pkg/earthly/earthly_test.go +++ b/forge/cli/pkg/earthly/earthly_test.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "maps" + "reflect" "slices" "testing" @@ -16,11 +17,11 @@ import ( func TestEarthlyExecutorRun(t *testing.T) { tests := []struct { - expect EarthlyExecutionResult name string output string earthlyExec EarthlyExecutor mockExec executor.ExecutorMock + expect map[string]EarthlyExecutionResult expectCalls int expectErr bool }{ @@ -36,12 +37,14 @@ Image foo output as bar Artifact foo output as bar`), nil }, }, - expect: EarthlyExecutionResult{ - Images: map[string]string{ - "foo": "bar", - }, - Artifacts: map[string]string{ - "foo": "bar", + expect: map[string]EarthlyExecutionResult{ + getNativePlatform(): { + Images: map[string]string{ + "foo": "bar", + }, + Artifacts: map[string]string{ + "foo": "bar", + }, }, }, expectErr: false, @@ -58,13 +61,44 @@ Artifact foo output as bar`), nil return []byte{}, fmt.Errorf("error") }, }, - expect: EarthlyExecutionResult{ - Images: map[string]string{}, - Artifacts: map[string]string{}, - }, + expect: nil, expectErr: true, expectCalls: 4, }, + { + name: "with platforms", + earthlyExec: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithPlatforms("foo", "bar"), + ), + mockExec: executor.ExecutorMock{ + ExecuteFunc: func(command string, args []string) ([]byte, error) { + return []byte(`foobarbaz +Image foo output as bar +Artifact foo output as bar`), nil + }, + }, + expect: map[string]EarthlyExecutionResult{ + "foo": { + Images: map[string]string{ + "foo": "bar", + }, + Artifacts: map[string]string{ + "foo": "bar", + }, + }, + "bar": { + Images: map[string]string{ + "foo": "bar", + }, + Artifacts: map[string]string{ + "foo": "bar", + }, + }, + }, + expectErr: false, + expectCalls: 2, + }, } for i := range tests { @@ -83,12 +117,8 @@ Artifact foo output as bar`), nil t.Errorf("expected %d calls to Execute, got %d", tt.expectCalls, len(tt.mockExec.ExecuteCalls())) } - if !maps.Equal(got.Artifacts, tt.expect.Artifacts) { - t.Errorf("expected %v, got %v", tt.expect.Artifacts, got.Artifacts) - } - - if !maps.Equal(got.Images, tt.expect.Images) { - t.Errorf("expected %v, got %v", tt.expect.Images, got.Images) + if !reflect.DeepEqual(got, tt.expect) { + t.Errorf("expected %v, got %v", tt.expect, got) } }) } @@ -96,16 +126,26 @@ Artifact foo output as bar`), nil func TestEarthlyExecutor_buildArguments(t *testing.T) { tests := []struct { - name string - e EarthlyExecutor - expect []string + name string + e EarthlyExecutor + platform string + expect []string }{ { name: "simple", e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, testutils.NewNoopLogger(), ), - expect: []string{"/test/dir+foo"}, + platform: getNativePlatform(), + expect: []string{"/test/dir+foo"}, + }, + { + name: "with platform", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + ), + platform: "foo/bar", + expect: []string{"--platform", "foo/bar", "/test/dir+foo"}, }, { name: "with target args", @@ -113,7 +153,8 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) { testutils.NewNoopLogger(), WithTargetArgs("--arg1", "foo", "--arg2", "bar"), ), - expect: []string{"/test/dir+foo", "--arg1", "foo", "--arg2", "bar"}, + platform: getNativePlatform(), + expect: []string{"/test/dir+foo", "--arg1", "foo", "--arg2", "bar"}, }, { name: "with artifact", @@ -121,7 +162,8 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) { testutils.NewNoopLogger(), WithArtifact("test"), ), - expect: []string{"--artifact", "/test/dir+foo/*", "test/"}, + platform: getNativePlatform(), + expect: []string{"--artifact", "/test/dir+foo/*", "test/"}, }, { name: "with ci", @@ -129,15 +171,8 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) { testutils.NewNoopLogger(), WithCI(), ), - expect: []string{"--ci", "/test/dir+foo"}, - }, - { - name: "with platform", - e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, - testutils.NewNoopLogger(), - WithPlatform("platform"), - ), - expect: []string{"--platform", "platform", "/test/dir+foo"}, + platform: getNativePlatform(), + expect: []string{"--ci", "/test/dir+foo"}, }, { name: "with privileged", @@ -145,7 +180,8 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) { testutils.NewNoopLogger(), WithPrivileged(), ), - expect: []string{"--allow-privileged", "/test/dir+foo"}, + platform: getNativePlatform(), + expect: []string{"--allow-privileged", "/test/dir+foo"}, }, { name: "with satellite", @@ -153,13 +189,14 @@ func TestEarthlyExecutor_buildArguments(t *testing.T) { testutils.NewNoopLogger(), WithSatellite("satellite"), ), - expect: []string{"--sat", "satellite", "/test/dir+foo"}, + platform: getNativePlatform(), + expect: []string{"--sat", "satellite", "/test/dir+foo"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := tt.e.buildArguments() + got := tt.e.buildArguments(tt.platform) if !slices.Equal(got, tt.expect) { t.Errorf("expected %v, got %v", tt.expect, got) } diff --git a/forge/cli/pkg/earthly/options.go b/forge/cli/pkg/earthly/options.go index e2578ad..49833dd 100644 --- a/forge/cli/pkg/earthly/options.go +++ b/forge/cli/pkg/earthly/options.go @@ -25,11 +25,11 @@ func WithCI() EarthlyExecutorOption { } } -// WithPlatform is an option for configuring an EarthlyExecutor to run the -// Earthly target with the given platform. -func WithPlatform(platform string) EarthlyExecutorOption { +// WithPlatforms is an option for configuring an EarthlyExecutor to run the +// Earthly target against the given platforms. +func WithPlatforms(platforms ...string) EarthlyExecutorOption { return func(e *EarthlyExecutor) { - e.earthlyArgs = append(e.earthlyArgs, "--platform", platform) + e.opts.platforms = append(e.opts.platforms, platforms...) } }