From a844d5e28dad3fbd65be215c01ecb76a9b91d9da Mon Sep 17 00:00:00 2001 From: Paul Annesley Date: Mon, 14 Nov 2022 22:36:34 +1030 Subject: [PATCH] Embedded JavaScript pipeline PoC --- clicommand/pipeline_eval.go | 217 ++++++++++++++++++++++ go.mod | 4 + go.sum | 22 +++ main.go | 1 + resources/node_modules/buildkite/hello.js | 1 + resources/resources.go | 8 + test/fixtures/pipelines/buildkite.js | 26 +++ 7 files changed, 279 insertions(+) create mode 100644 clicommand/pipeline_eval.go create mode 100644 resources/node_modules/buildkite/hello.js create mode 100644 resources/resources.go create mode 100644 test/fixtures/pipelines/buildkite.js diff --git a/clicommand/pipeline_eval.go b/clicommand/pipeline_eval.go new file mode 100644 index 0000000000..f73e10489e --- /dev/null +++ b/clicommand/pipeline_eval.go @@ -0,0 +1,217 @@ +package clicommand + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/buildkite/agent/v3/cliconfig" + "github.com/buildkite/agent/v3/resources" + "github.com/buildkite/agent/v3/stdin" + "github.com/buildkite/yaml" + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/console" + "github.com/dop251/goja_nodejs/process" + "github.com/dop251/goja_nodejs/require" + "github.com/urfave/cli" +) + +const evalDescription = `Usage: + buildkite-agent pipeline eval [options] + +Description: + Something something JavaScript? + +Example: + $ buildkite-agent pipeline eval buildkite.js + + Evaluates buildkite.js as JavaScript and perhaps uploads the stdout as JSON/YAML pipeline? +` + +type PipelineEvalConfig struct { + FilePath string `cli:"arg:0" label:"upload paths"` + + // Global flags + Debug bool `cli:"debug"` + LogLevel string `cli:"log-level"` + NoColor bool `cli:"no-color"` + Experiments []string `cli:"experiment" normalize:"list"` + Profile string `cli:"profile"` +} + +var PipelineEvalCommand = cli.Command{ + Name: "eval", + Usage: "Evaluates a JavaScript pipeline", + Description: evalDescription, + Flags: []cli.Flag{ + // Global flags + NoColorFlag, + DebugFlag, + LogLevelFlag, + ExperimentsFlag, + ProfileFlag, + }, + Action: func(c *cli.Context) error { + // The configuration will be loaded into this struct + cfg := PipelineEvalConfig{} + + loader := cliconfig.Loader{CLI: c, Config: &cfg} + warnings, err := loader.Load() + if err != nil { + fmt.Printf("%s", err) + os.Exit(1) + } + + l := CreateLogger(&cfg) + + // Now that we have a logger, log out the warnings that loading config generated + for _, warning := range warnings { + l.Warn("%s", warning) + } + + // Setup any global configuration options + done := HandleGlobalFlags(l, cfg) + defer done() + + // Find the pipeline file either from STDIN or the first + // argument + var input []byte + var filename string + + if cfg.FilePath != "" { + l.Info("Reading pipeline config from \"%s\"", cfg.FilePath) + + filename = filepath.Base(cfg.FilePath) + input, err = os.ReadFile(cfg.FilePath) + if err != nil { + l.Fatal("Failed to read file: %s", err) + } + } else if stdin.IsReadable() { + l.Info("Reading pipeline config from STDIN") + + // Actually read the file from STDIN + input, err = io.ReadAll(os.Stdin) + if err != nil { + l.Fatal("Failed to read from STDIN: %s", err) + } + } else { + l.Info("Searching for pipeline config...") + + paths := []string{ + "buildkite.js", + filepath.FromSlash(".buildkite/buildkite.js"), + filepath.FromSlash("buildkite/buildkite.js"), + } + + // Collect all the files that exist + exists := []string{} + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + exists = append(exists, path) + } + } + + // If more than 1 of the config files exist, throw an + // error. There can only be one!! + if len(exists) > 1 { + l.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", ")) + } else if len(exists) == 0 { + l.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.") + } + + found := exists[0] + + l.Info("Found config file \"%s\"", found) + + // Read the default file + filename = path.Base(found) + input, err = os.ReadFile(found) + if err != nil { + l.Fatal("Failed to read file \"%s\" (%s)", found, err) + } + } + + if err := evalJS(filename, input, c.App.Writer); err != nil { + panic(err) + } + return nil + }, +} + +func evalJS(filename string, input []byte, output io.Writer) error { + runtime := goja.New() + runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true)) + + // Add support for require() CommonJS modules. + // require("buildkite/*") is handled by embedded resources/node_modules/buildkite/* filesystem. + // Other paths are loaded from the host filesystem. + registry := require.NewRegistry( + require.WithLoader(func(name string) ([]byte, error) { + if !strings.HasPrefix(name, "node_modules/buildkite/") { + return require.DefaultSourceLoader(name) + } + res := resources.FS + data, err := res.ReadFile(name) + if errors.Is(err, fs.ErrNotExist) { + return nil, require.ModuleFileDoesNotExistError + } else if err != nil { + return nil, err + } + return data, nil + }), + ) + registry.Enable(runtime) + + // Add basic utilities + console.Enable(runtime) // console.log() + process.Enable(runtime) // process.env() + + // provide plugin() as a native module (implemented in Go) + registry.RegisterNativeModule("buildkite/plugin", func(runtime *goja.Runtime, module *goja.Object) { + module.Set("exports", func(call goja.FunctionCall) goja.Value { + name := call.Argument(0) + ref := call.Argument(1) + config := call.Argument(2) + plugin := runtime.NewObject() + plugin.Set(name.String()+"#"+ref.String(), config) + return plugin + }) + }) + + // provide assignable module.exports for Pipeline result + rootModule := runtime.NewObject() + rootModule.Set("exports", runtime.NewObject()) + err := runtime.Set("module", rootModule) + if err != nil { + return err + } + + if filename == "" { + filename = "(stdin)" + } + + v, err := runtime.RunScript(filename, string(input)) + if err != nil { + panic(err) + } + + y, err := yaml.Marshal(v.Export()) + if err != nil { + return err + } + + n, err := output.Write(y) + if err != nil { + return nil + } + if n != len(y) { + return errors.New("short write") + } + + return nil +} diff --git a/go.mod b/go.mod index 437cf40aab..901cf0e87a 100644 --- a/go.mod +++ b/go.mod @@ -58,9 +58,13 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 // indirect + github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 0a4e626ba4..9b54411203 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,7 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -116,6 +117,17 @@ github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/Lu github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= +github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 h1:arM6Tq1Ba+a9FWuq3S6Qgrfd5MD0slQdMnCKI2VclFg= +github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 h1:p3QZwRRfCN7Qr3GNBTMKBkLFjEm3DHR4MaJABvsiqgk= +github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6/go.mod h1:+CJy9V5cGycP5qwp6RM5jLg+TFEMyGtD7A9xUbU/BOQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= @@ -140,6 +152,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -251,9 +265,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU= @@ -285,6 +303,7 @@ github.com/rjeczalik/interfaces v0.1.1 h1:xhFQNGtz3T3CQgtJJwWn+i3Ekl1WeObh7wtTtC github.com/rjeczalik/interfaces v0.1.1/go.mod h1:TNwD+kCGmXYrXksRDD5ikspp08m/Aosbr67zVLMjnOY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.0.0-20180226215254-237a9547c8a5 h1:T7hUw7pBSINuHQyWwMdfIWZZH5M3ju4yXIbuV/Upp+4= @@ -811,11 +830,14 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.43.1/go.mod h1:YL9g+nlUY7ByCffD5pDytAqy99GNby gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 595e93be47..8d1080d95b 100644 --- a/main.go +++ b/main.go @@ -95,6 +95,7 @@ func main() { Usage: "Make changes to the pipeline of the currently running build", Subcommands: []cli.Command{ clicommand.PipelineUploadCommand, + clicommand.PipelineEvalCommand, }, }, { diff --git a/resources/node_modules/buildkite/hello.js b/resources/node_modules/buildkite/hello.js new file mode 100644 index 0000000000..a67fa04b25 --- /dev/null +++ b/resources/node_modules/buildkite/hello.js @@ -0,0 +1 @@ +console.log("hello embedded FS!"); diff --git a/resources/resources.go b/resources/resources.go new file mode 100644 index 0000000000..153ee10d14 --- /dev/null +++ b/resources/resources.go @@ -0,0 +1,8 @@ +package resources + +import "embed" + +// FS is an embedded filesystem. +// +//go:embed node_modules +var FS embed.FS diff --git a/test/fixtures/pipelines/buildkite.js b/test/fixtures/pipelines/buildkite.js new file mode 100644 index 0000000000..9ffc5dde9e --- /dev/null +++ b/test/fixtures/pipelines/buildkite.js @@ -0,0 +1,26 @@ +plugin = require("buildkite/plugin"); +require("buildkite/hello"); + +const dockerCompose = plugin("docker-compose", "v3.0.0", { + config: ".buildkite/docker-compose.yml", + run: "agent", +}); + +pipeline = { + env: { + DRY_RUN: !!process.env.DRY_RUN, + }, + agents: { + queue: "agent-runners-linux-amd64", + }, + steps: [ + { + name: ":go: go fmt", + key: "test-go-fmt", + command: ".buildkite/steps/test-go-fmt.sh", + plugins: [dockerCompose], + }, + ], +}; + +module.exports = pipeline;