From ba9c3e76d88470ccf2784bb23458144d211c3fdd Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 1 Apr 2024 13:54:57 -0500 Subject: [PATCH] feat: add support for configuring/injecting cloud-init into VMs Update to qcli v0.3.1 which added support for a vvfatt driver enabling machine to create and attach a directory as a VFAT disk in the guest. Add a new CloudConfig portion to the API that can import and use user-data, meta-data and network-config. Definiting these values in VM's config will render out the defined user-data, network-config and meta-data (auto generated if not supplied). This device is detected as as NoCloud data source and will initilized the guest if cloud-init is present in the image. Additional changes: - moved utility functions to pkg/api/utils.go - added unittests for cloud-init API - Added 'test' and 'test-api' make targets - Added example configuration with cloud-init - Updated README to point to examples directory - Fixes some missing arguments to prints/logs found by go test Signed-off-by: Ryan Harper --- Makefile | 6 + README.md | 5 + doc/examples/vm-with-cloud-init.yaml | 25 +++ go.mod | 7 +- go.sum | 9 +- pkg/api/cloudinit.go | 202 ++++++++++++++++++++ pkg/api/cloudinit_test.go | 136 ++++++++++++++ pkg/api/controller.go | 240 +----------------------- pkg/api/qconfig.go | 14 ++ pkg/api/util.go | 265 +++++++++++++++++++++++++++ pkg/api/vm.go | 84 +++++++-- 11 files changed, 730 insertions(+), 263 deletions(-) create mode 100644 doc/examples/vm-with-cloud-init.yaml create mode 100644 pkg/api/cloudinit.go create mode 100644 pkg/api/cloudinit_test.go create mode 100644 pkg/api/util.go diff --git a/Makefile b/Makefile index dff5ac3..4e76af3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ BINS := bin/machine bin/machined .PHONY: all clean all: $(BINS) +.PHONY: test +test: test-api + clean: rm -f -v $(BINS) @@ -11,3 +14,6 @@ bin/machine: cmd/machine/cmd/*.go pkg/*/*.go bin/machined: cmd/machined/cmd/*.go pkg/*/*.go go build -o $@ cmd/machined/cmd/*.go + +test-api: + go test pkg/api/*.go diff --git a/README.md b/README.md index de4292c..6b6a445 100644 --- a/README.md +++ b/README.md @@ -132,3 +132,8 @@ $ bin/machine start vm1 200 OK $ bin/machine gui vm1 ``` + + +## Examples + +See [doc/examples](doc/examples/) for other example VM definitions. diff --git a/doc/examples/vm-with-cloud-init.yaml b/doc/examples/vm-with-cloud-init.yaml new file mode 100644 index 0000000..d731a9a --- /dev/null +++ b/doc/examples/vm-with-cloud-init.yaml @@ -0,0 +1,25 @@ +name: f40-vm1 +type: kvm +ephemeral: false +description: Fedora 40 Beta with UKI +config: + name: f40-vm1 + uefi: true + tpm: true + gui: false + tpm-version: 2.0 + secure-boot: false + uefi-code: /usr/share/OVMF/OVMF_CODE.fd + disks: + - file: import/Fedora-Cloud-Base-UEFI-UKI.x86_64-40-1.10.qcow2 + type: ssd + format: qcow2 + cloud-init: + user-data: | + #cloud-config + password: + chpasswd: { expire: False } + ssh_pwauth: True + ssh-authorized-keys: + - | + ssh-ed25519 xxxxx diff --git a/go.mod b/go.mod index 3658899..b4235fe 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,17 @@ require ( github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 github.com/gin-gonic/gin v1.8.1 github.com/go-resty/resty/v2 v2.7.0 + github.com/google/uuid v1.3.0 github.com/lxc/lxd v0.0.0-20221130220346-2c77027b7a5e github.com/mitchellh/go-homedir v1.1.0 github.com/msoap/byline v1.1.1 - github.com/project-machine/qcli v0.2.1 + github.com/pkg/errors v0.9.1 + github.com/project-machine/qcli v0.3.1 github.com/rodaine/table v1.1.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.14.0 - golang.org/x/sys v0.2.0 + golang.org/x/sys v0.5.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -28,7 +30,6 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect github.com/goccy/go-json v0.9.7 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect diff --git a/go.sum b/go.sum index 56193aa..c82e51c 100644 --- a/go.sum +++ b/go.sum @@ -204,14 +204,15 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-machine/qcli v0.2.1 h1:rIRItjdkeBbD4NIxYyTkxCeJIolGHdniJ51Phfg2Ols= -github.com/project-machine/qcli v0.2.1/go.mod h1:N7+8pGJWD/PvJOmzY6dmor4DYnG18bW/Mwa5Ful0GEs= +github.com/project-machine/qcli v0.3.1 h1:OIiLUZa6acaFgtT/Ev//cehxblBSEtsqBNSNGVnyrc4= +github.com/project-machine/qcli v0.3.1/go.mod h1:DEmDcRYrSL4s1DuYY7DxL3vEW6fMiVAQ6Ku7Fi1lOg8= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -404,8 +405,8 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= diff --git a/pkg/api/cloudinit.go b/pkg/api/cloudinit.go new file mode 100644 index 0000000..59dcb98 --- /dev/null +++ b/pkg/api/cloudinit.go @@ -0,0 +1,202 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package api + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/google/uuid" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +const ( + NoCloudFSLabel = "cidata" +) + +/* +type: kvm +config: + + name: slick-seal + ... + cloud-init: + user-data:| + #cloud-config + runcmd: + - cat /etc/os-release + network-config:| + version: 2 + ethernets: + nic0: + match: + name: en* + dhcp4: true + meta-data:| + instance-id: 08b2083d-2935-4d50-a442-d1da8920de20 + local-hostname: slick-seal +*/ + +type CloudInitConfig struct { + NetworkConfig string `yaml:"network-config"` + UserData string `yaml:"user-data"` + MetaData string `yaml:"meta-data"` +} + +type MetaData struct { + InstanceId string `yaml:"instance-id"` + LocalHostname string `yaml:"local-hostname"` +} + +func HasCloudConfig(config CloudInitConfig) bool { + + if config.MetaData != "" { + return true + } + if config.UserData != "" { + return true + } + if config.NetworkConfig != "" { + return true + } + return false +} + +func PrepareMetadata(config *CloudInitConfig, hostname string) error { + // update MetaData with local-hostname and instance-id if not set + + if config.MetaData != "" { + return fmt.Errorf("cloud-init config has existing metadata") + } + + iid := uuid.New() + + md := MetaData{ + InstanceId: iid.String(), + LocalHostname: hostname, + } + + content, err := yaml.Marshal(&md) + if err != nil { + return fmt.Errorf("failed to marshal metadata: %s", err) + } + + config.MetaData = string(content) + + return nil +} + +func RenderCloudInitConfig(config CloudInitConfig, outputPath string) error { + + renderedFiles := 0 + for _, d := range []struct { + confFile string + confData string + }{ + { + confFile: "network-config", + confData: config.NetworkConfig, + }, + { + confFile: "user-data", + confData: config.UserData, + }, + { + confFile: "meta-data", + confData: config.MetaData, + }, + } { + if len(d.confData) > 0 { + configFile := filepath.Join(outputPath, d.confFile) + tempFile, err := os.CreateTemp("", "tmp-cloudinit-") + if err != nil { + return fmt.Errorf("failed to create a temp file for writing cloud-init %s file: %s", d.confFile, err) + } + defer tempFile.Close() + defer os.Remove(tempFile.Name()) + if err := os.WriteFile(tempFile.Name(), []byte(d.confData), 0666); err != nil { + return fmt.Errorf("failed to write cloud-init %s file %q: %s", d.confFile, tempFile.Name(), err) + } + if err := os.Rename(tempFile.Name(), configFile); err != nil { + return fmt.Errorf("failed to rename temp file %q to %q: %s", tempFile.Name(), configFile, err) + } + renderedFiles++ + } + } + if renderedFiles == 0 { + return fmt.Errorf("failed to render any cloud-init config files; maybe empty cloud-init config?") + } + return nil +} + +func verifyCloudInitConfig(cfg CloudInitConfig, contentsDir string) error { + + // read the extracted directory and validate CloudInitConfig files + err := filepath.Walk(contentsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + log.Infof("verifyCloudInitCfg: path:%s name:%s contents:%s", path, info.Name(), contents) + switch info.Name() { + case "network-config": + if cfg.NetworkConfig != string(contents) { + return fmt.Errorf("network-config: expected contents %q, got %q", cfg.NetworkConfig, string(contents)) + } + case "user-data": + if cfg.UserData != string(contents) { + return fmt.Errorf("user-data: expected contents %q, got %q", cfg.UserData, string(contents)) + } + case "meta-data": + if cfg.MetaData != string(contents) { + return fmt.Errorf("meta-data: expected contents %q, got %q", cfg.MetaData, string(contents)) + } + default: + return fmt.Errorf("Unexpected file %q in cloud-init rendered directory", info.Name()) + } + } else { + if info.Name() != filepath.Base(path) { + return fmt.Errorf("Unexpected directory %q in cloud-init rendered directory", info.Name()) + } + } + + return nil + }) + + return err +} + +func CreateLocalDataSource(cfg CloudInitConfig, directory string) error { + + if err := EnsureDir(directory); err != nil { + return fmt.Errorf("failed to create cloud-init data source directory %q: %s", directory, err) + } + + if err := RenderCloudInitConfig(cfg, directory); err != nil { + return fmt.Errorf("failed to render cloud-init config to directory %q: %s", directory, err) + } + + if err := verifyCloudInitConfig(cfg, directory); err != nil { + return fmt.Errorf("failed to verify cloud-init config content in directory %q: %s", directory, err) + } + + return nil +} diff --git a/pkg/api/cloudinit_test.go b/pkg/api/cloudinit_test.go new file mode 100644 index 0000000..ea0bdd1 --- /dev/null +++ b/pkg/api/cloudinit_test.go @@ -0,0 +1,136 @@ +package api + +import ( + "os" + "testing" +) + +func TestCloudInitRendersConfig(t *testing.T) { + + cfg := CloudInitConfig{ + NetworkConfig: "network-config", + UserData: "user-data", + MetaData: "meta-data", + } + + tmpDir, err := os.MkdirTemp("", "test-ci-render-config") + if err != nil { + t.Fatalf("failed to create a tempdir for test") + } + defer os.RemoveAll(tmpDir) + + err = RenderCloudInitConfig(cfg, tmpDir) + if err != nil { + t.Fatalf("unexpected error when rendering cloud-init config: %s", err) + } + + err = verifyCloudInitConfig(cfg, tmpDir) + if err != nil { + t.Fatalf("failed to verify rendered contents: %s", err) + } +} + +func TestCloudInitRenderConfigFailsOnEmpty(t *testing.T) { + + cfg := CloudInitConfig{ + NetworkConfig: "", + UserData: "", + MetaData: "", + } + + tmpDir, err := os.MkdirTemp("", "test-ci-render-config") + if err != nil { + t.Fatalf("failed to create a tempdir for test") + } + defer os.RemoveAll(tmpDir) + + err = RenderCloudInitConfig(cfg, tmpDir) + if err == nil { + t.Fatalf("expected empty config to return an error, got nil instead") + } +} + +func TestPrepareMetadataUpdatesConfig(t *testing.T) { + vmCfg := VMDef{ + Name: "myVM1", + CloudInit: CloudInitConfig{ + NetworkConfig: "network-config", + UserData: "user-data", + MetaData: "", + }, + } + + err := PrepareMetadata(&vmCfg.CloudInit, vmCfg.Name) + if err != nil { + t.Fatalf("failed to prepare metadata: %s", err) + } + + // log.Infof("vmCfg: %+v", vmCfg) + if vmCfg.CloudInit.MetaData == "" { + t.Fatalf("failed to update metadata, it's empty") + } + + tmpDir, err := os.MkdirTemp("", "test-ci-render-config") + if err != nil { + t.Fatalf("failed to create a tempdir for test") + } + defer os.RemoveAll(tmpDir) + + err = RenderCloudInitConfig(vmCfg.CloudInit, tmpDir) + if err != nil { + t.Fatalf("unexpected error when rendering cloud-init config: %s", err) + } +} + +func TestCloudInitCreatesDataSource(t *testing.T) { + + cfg := CloudInitConfig{ + NetworkConfig: "network-config", + UserData: "user-data", + MetaData: "meta-data", + } + + seedDir, err := os.MkdirTemp("", "test-ci-seed") + if err != nil { + t.Fatalf("failed to create a tempdir for test") + } + defer os.RemoveAll(seedDir) + + err = CreateLocalDataSource(cfg, seedDir) + if err != nil { + t.Fatalf("failed to cloud-init datasource: %s", err) + } + +} + +func TestPrepareMetadataUpdatesPresentInDataSource(t *testing.T) { + vmCfg := VMDef{ + Name: "myVM1", + CloudInit: CloudInitConfig{ + NetworkConfig: "network-config", + UserData: "user-data", + MetaData: "", + }, + } + + err := PrepareMetadata(&vmCfg.CloudInit, vmCfg.Name) + if err != nil { + t.Fatalf("failed to prepare metadata: %s", err) + } + + seedDir, err := os.MkdirTemp("", "test-ci-seed") + if err != nil { + t.Fatalf("failed to create a tempdir for test") + } + defer os.RemoveAll(seedDir) + + // log.Infof("vmCfg: %+v", vmCfg) + if vmCfg.CloudInit.MetaData == "" { + t.Fatalf("failed to update metadata, it's empty") + } + + err = CreateLocalDataSource(vmCfg.CloudInit, seedDir) + if err != nil { + t.Fatalf("failed to create data source: %s", err) + } +} diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 3e2aabb..b056aaf 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -1,10 +1,9 @@ /* - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -15,24 +14,16 @@ limitations under the License. package api import ( - "bytes" "context" "fmt" - "io" - "io/ioutil" "net" "net/http" "os" - "os/exec" "path/filepath" - "strings" "sync" - "syscall" - "time" "github.com/coreos/go-systemd/activation" "github.com/gin-gonic/gin" - "github.com/msoap/byline" log "github.com/sirupsen/logrus" ) @@ -149,232 +140,3 @@ func (c *Controller) Shutdown(ctx context.Context) error { } return nil } - -// -// utility functions below here -// - -func PathExists(d string) bool { - _, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - return true -} - -func WaitForPath(path string, retries, sleepSeconds int) bool { - var numRetries int - if retries == 0 { - numRetries = 1 - } else { - numRetries = retries - } - for i := 0; i < numRetries; i++ { - if PathExists(path) { - return true - } - time.Sleep(time.Duration(sleepSeconds) * time.Second) - } - return PathExists(path) -} - -func EnsureDir(dir string) error { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("couldn't make dirs: %s", err) - } - return nil -} - -func Which(commandName string) string { - return WhichInRoot(commandName, "") -} - -func WhichInRoot(commandName string, root string) string { - cmd := []string{"sh", "-c", "command -v \"$0\"", commandName} - if root != "" && root != "/" { - cmd = append([]string{"chroot", root}, cmd...) - } - out, rc := RunCommandWithRc(cmd...) - if rc == 0 { - return strings.TrimSuffix(string(out), "\n") - } - if rc != 127 { - log.Warnf("checking for %s exited unexpected value %d\n", commandName, rc) - } - return "" -} - -func LogCommand(args ...string) error { - return LogCommandWithFunc(log.Infof, args...) -} - -func LogCommandDebug(args ...string) error { - return LogCommandWithFunc(log.Debugf, args...) -} - -func LogCommandWithFunc(logf func(string, ...interface{}), args ...string) error { - cmd := exec.Command(args[0], args[1:]...) - stdoutPipe, err := cmd.StdoutPipe() - if err != nil { - logf("%s-fail | %s", err) - return err - } - cmd.Stderr = cmd.Stdout - err = cmd.Start() - if err != nil { - logf("%s-fail | %s", args[0], err) - return err - } - pid := cmd.Process.Pid - logf("|%d-start| %q", pid, args) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - err := byline.NewReader(stdoutPipe).Each( - func(line []byte) { - logf("|%d-out | %s", pid, line[:len(line)-1]) - }).Discard() - if err != nil { - log.Fatalf("Unexpected %s", err) - } - wg.Done() - }() - - wg.Wait() - err = cmd.Wait() - - logf("|%d-exit | rc=%d", pid, GetCommandErrorRC(err)) - return err -} - -// CopyFileBits - copy file content from a to b -// differs from CopyFile in: -// - does not do permissions - new files created with 0644 -// - if src is a symlink, copies content, not link. -// - does not invoke sh. -func CopyFileBits(src, dest string) error { - if len(src) == 0 { - return fmt.Errorf("Source file is empty string") - } - if len(dest) == 0 { - return fmt.Errorf("Destination file is empty string") - } - in, err := os.Open(src) - if err != nil { - return fmt.Errorf("Failed to open source file %q: %s", src, err) - } - defer in.Close() - - out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("Failed to open destination file %q", dest, err) - } - defer out.Close() - - _, err = io.Copy(out, in) - if err != nil { - return fmt.Errorf("Failed while copying %q -> %q: %s", src, dest, err) - } - return out.Close() -} - -// Copy one file to a new path, i.e. cp a b -func CopyFileRefSparse(src, dest string) error { - if err := EnsureDir(filepath.Dir(src)); err != nil { - return err - } - if err := EnsureDir(filepath.Dir(dest)); err != nil { - return err - } - cmdtxt := fmt.Sprintf("cp --force --reflink=auto --sparse=auto %s %s", src, dest) - return RunCommand("sh", "-c", cmdtxt) -} - -func RunCommand(args ...string) error { - cmd := exec.Command(args[0], args[1:]...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%s: %s: %s", strings.Join(args, " "), err, string(output)) - } - return nil -} - -func RunCommandWithRc(args ...string) ([]byte, int) { - out, err := exec.Command(args[0], args[1:]...).CombinedOutput() - return out, GetCommandErrorRC(err) -} - -func RunCommandWithOutputErrorRc(args ...string) ([]byte, []byte, int) { - cmd := exec.Command(args[0], args[1:]...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - return stdout.Bytes(), stderr.Bytes(), GetCommandErrorRC(err) -} - -func GetCommandErrorRCDefault(err error, rcError int) int { - if err == nil { - return 0 - } - exitError, ok := err.(*exec.ExitError) - if ok { - if status, ok := exitError.Sys().(syscall.WaitStatus); ok { - return status.ExitStatus() - } - } - log.Debugf("Unavailable return code for %s. returning %d", err, rcError) - return rcError -} - -func GetCommandErrorRC(err error) int { - return GetCommandErrorRCDefault(err, 127) -} - -func GetTempSocketDir() (string, error) { - d, err := ioutil.TempDir("/tmp", "msockets-*") - if err != nil { - return "", nil - } - if err := checkSocketDir(d); err != nil { - os.RemoveAll(d) - return "", err - } - return d, nil -} - -// LinuxUnixSocketMaxLen - 108 chars max for a unix socket path (including null byte). -const LinuxUnixSocketMaxLen int = 108 - -func checkSocketDir(sdir string) error { - // just use this as a filename that might go there. - fname := "monitor.socket" - if len(sdir)+len(fname) >= LinuxUnixSocketMaxLen { - return fmt.Errorf("dir %s is too long (%d) to hold a unix socket", sdir, len(sdir)) - } - return nil -} - -func ForceLink(oldname, newname string) error { - if oldname == "" { - return fmt.Errorf("empty string for parameter 'oldname'") - } - if newname == "" { - return fmt.Errorf("empty string for parameter 'newname'") - } - if !PathExists(oldname) { - return fmt.Errorf("Source file %s does not exist", oldname) - } - log.Debugf("forceLink oldname=%s newname=%s", oldname, newname) - if err := os.Remove(newname); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("Failed removing %s before linking to %s: %s", newname, oldname, err) - } - if err := os.Symlink(oldname, newname); err != nil { - return fmt.Errorf("Failed linking %s -> %s: %s", oldname, newname, err) - } - if !PathExists(newname) { - return fmt.Errorf("Failed to symlink %s -> %s", newname, oldname) - } - return nil -} diff --git a/pkg/api/qconfig.go b/pkg/api/qconfig.go index 6e307c8..49932e7 100644 --- a/pkg/api/qconfig.go +++ b/pkg/api/qconfig.go @@ -402,6 +402,20 @@ func ConfigureUEFIVars(c *qcli.Config, srcCode, srcVars, runDir string, secureBo return nil } +func NewVVFATBlockDev(id, directory, label string) (qcli.BlockDevice, error) { + blkdev := qcli.BlockDevice{ + Driver: qcli.VVFAT, + ID: id, + VVFATDev: qcli.VVFATDev{ + Driver: qcli.VirtioBlock, + Directory: directory, + Label: label, + FATMode: qcli.FATMode16, + }, + } + return blkdev, nil +} + func GenerateQConfig(runDir, sockDir string, v VMDef) (*qcli.Config, error) { var c *qcli.Config var err error diff --git a/pkg/api/util.go b/pkg/api/util.go new file mode 100644 index 0000000..1a3ba49 --- /dev/null +++ b/pkg/api/util.go @@ -0,0 +1,265 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package api + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/msoap/byline" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func PathExists(d string) bool { + _, err := os.Stat(d) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +func WaitForPath(path string, retries, sleepSeconds int) bool { + var numRetries int + if retries == 0 { + numRetries = 1 + } else { + numRetries = retries + } + for i := 0; i < numRetries; i++ { + if PathExists(path) { + return true + } + time.Sleep(time.Duration(sleepSeconds) * time.Second) + } + return PathExists(path) +} + +func EnsureDir(dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("couldn't make dirs: %s", err) + } + return nil +} + +func Which(commandName string) string { + return WhichInRoot(commandName, "") +} + +func WhichInRoot(commandName string, root string) string { + cmd := []string{"sh", "-c", "command -v \"$0\"", commandName} + if root != "" && root != "/" { + cmd = append([]string{"chroot", root}, cmd...) + } + out, rc := RunCommandWithRc(cmd...) + if rc == 0 { + return strings.TrimSuffix(string(out), "\n") + } + if rc != 127 { + log.Warnf("checking for %s exited unexpected value %d\n", commandName, rc) + } + return "" +} + +func LogCommand(args ...string) error { + return LogCommandWithFunc(log.Infof, args...) +} + +func LogCommandDebug(args ...string) error { + return LogCommandWithFunc(log.Debugf, args...) +} + +func LogCommandWithFunc(logf func(string, ...interface{}), args ...string) error { + cmd := exec.Command(args[0], args[1:]...) + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + logf("%s-fail | %s", err) + return err + } + cmd.Stderr = cmd.Stdout + err = cmd.Start() + if err != nil { + logf("%s-fail | %s", args[0], err) + return err + } + pid := cmd.Process.Pid + logf("|%d-start| %q", pid, args) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + err := byline.NewReader(stdoutPipe).Each( + func(line []byte) { + logf("|%d-out | %s", pid, line[:len(line)-1]) + }).Discard() + if err != nil { + log.Fatalf("Unexpected %s", err) + } + wg.Done() + }() + + wg.Wait() + err = cmd.Wait() + + logf("|%d-exit | rc=%d", pid, GetCommandErrorRC(err)) + return err +} + +// CopyFileBits - copy file content from a to b +// differs from CopyFile in: +// - does not do permissions - new files created with 0644 +// - if src is a symlink, copies content, not link. +// - does not invoke sh. +func CopyFileBits(src, dest string) error { + if len(src) == 0 { + return fmt.Errorf("Source file is empty string") + } + if len(dest) == 0 { + return fmt.Errorf("Destination file is empty string") + } + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("Failed to open source file %q: %s", src, err) + } + defer in.Close() + + out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("Failed to open destination file %q: %s", dest, err) + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return fmt.Errorf("Failed while copying %q -> %q: %s", src, dest, err) + } + return out.Close() +} + +// Copy one file to a new path, i.e. cp a b +func CopyFileRefSparse(src, dest string) error { + if err := EnsureDir(filepath.Dir(src)); err != nil { + return err + } + if err := EnsureDir(filepath.Dir(dest)); err != nil { + return err + } + cmdtxt := fmt.Sprintf("cp --force --reflink=auto --sparse=auto %s %s", src, dest) + return RunCommand("sh", "-c", cmdtxt) +} + +func RsyncDirWithErrorQuiet(src, dest string) error { + err := LogCommand("rsync", "--quiet", "--archive", src+"/", dest+"/") + if err != nil { + return errors.Wrapf(err, "Failed copying %s to %s\n", src, dest) + } + return nil +} + +func RunCommand(args ...string) error { + cmd := exec.Command(args[0], args[1:]...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %s: %s", strings.Join(args, " "), err, string(output)) + } + return nil +} + +func RunCommandWithRc(args ...string) ([]byte, int) { + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + return out, GetCommandErrorRC(err) +} + +func RunCommandWithOutputErrorRc(args ...string) ([]byte, []byte, int) { + cmd := exec.Command(args[0], args[1:]...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.Bytes(), stderr.Bytes(), GetCommandErrorRC(err) +} + +func GetCommandErrorRCDefault(err error, rcError int) int { + if err == nil { + return 0 + } + exitError, ok := err.(*exec.ExitError) + if ok { + if status, ok := exitError.Sys().(syscall.WaitStatus); ok { + return status.ExitStatus() + } + } + log.Debugf("Unavailable return code for %s. returning %d", err, rcError) + return rcError +} + +func GetCommandErrorRC(err error) int { + return GetCommandErrorRCDefault(err, 127) +} + +func GetTempSocketDir() (string, error) { + d, err := ioutil.TempDir("/tmp", "msockets-*") + if err != nil { + return "", nil + } + if err := checkSocketDir(d); err != nil { + os.RemoveAll(d) + return "", err + } + return d, nil +} + +// LinuxUnixSocketMaxLen - 108 chars max for a unix socket path (including null byte). +const LinuxUnixSocketMaxLen int = 108 + +func checkSocketDir(sdir string) error { + // just use this as a filename that might go there. + fname := "monitor.socket" + if len(sdir)+len(fname) >= LinuxUnixSocketMaxLen { + return fmt.Errorf("dir %s is too long (%d) to hold a unix socket", sdir, len(sdir)) + } + return nil +} + +func ForceLink(oldname, newname string) error { + if oldname == "" { + return fmt.Errorf("empty string for parameter 'oldname'") + } + if newname == "" { + return fmt.Errorf("empty string for parameter 'newname'") + } + if !PathExists(oldname) { + return fmt.Errorf("Source file %s does not exist", oldname) + } + log.Debugf("forceLink oldname=%s newname=%s", oldname, newname) + if err := os.Remove(newname); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("Failed removing %s before linking to %s: %s", newname, oldname, err) + } + if err := os.Symlink(oldname, newname); err != nil { + return fmt.Errorf("Failed linking %s -> %s: %s", oldname, newname, err) + } + if !PathExists(newname) { + return fmt.Errorf("Failed to symlink %s -> %s", newname, oldname) + } + return nil +} diff --git a/pkg/api/vm.go b/pkg/api/vm.go index 934fc18..d625190 100644 --- a/pkg/api/vm.go +++ b/pkg/api/vm.go @@ -57,20 +57,21 @@ func (v VMState) String() string { } type VMDef struct { - Name string `yaml:"name"` - Cpus uint32 `yaml:"cpus" default:1` - Memory uint32 `yaml:"memory"` - Serial string `yaml:"serial"` - Nics []NicDef `yaml:"nics"` - Disks []QemuDisk `yaml:"disks"` - Boot string `yaml:"boot"` - Cdrom string `yaml:"cdrom"` - UEFICode string `yaml:"uefi-code"` - UEFIVars string `yaml:"uefi-vars"` - TPM bool `yaml:"tpm"` - TPMVersion string `yaml:"tpm-version"` - SecureBoot bool `yaml:"secure-boot"` - Gui bool `yaml:"gui"` + Name string `yaml:"name"` + Cpus uint32 `yaml:"cpus" default:1` + Memory uint32 `yaml:"memory"` + Serial string `yaml:"serial"` + Nics []NicDef `yaml:"nics"` + Disks []QemuDisk `yaml:"disks"` + Boot string `yaml:"boot"` + Cdrom string `yaml:"cdrom"` + UEFICode string `yaml:"uefi-code"` + UEFIVars string `yaml:"uefi-vars"` + TPM bool `yaml:"tpm"` + TPMVersion string `yaml:"tpm-version"` + SecureBoot bool `yaml:"secure-boot"` + Gui bool `yaml:"gui"` + CloudInit CloudInitConfig `yaml:"cloud-init"` } func (v *VMDef) adjustDiskBootIdx(qti *qcli.QemuTypeIndex) ([]string, error) { @@ -253,6 +254,55 @@ func newVM(ctx context.Context, clusterName string, vmConfig VMDef) (*VM, error) return &VM{}, fmt.Errorf("Failed to generate qcli Config from VM definition: %s", err) } + // generate cloud-init seed dir if config is present + if HasCloudConfig(vmConfig.CloudInit) { + log.Infof("newVM: vm has cloud-init config, generating ci data") + log.Infof("newVM: network-config: %s", vmConfig.CloudInit.NetworkConfig) + log.Infof("newVM: user-data: %s", vmConfig.CloudInit.UserData) + log.Infof("newVM: meta-data: %s", vmConfig.CloudInit.MetaData) + + // insert MetaData if needed + if vmConfig.CloudInit.MetaData == "" { + log.Infof("newVM: preparing metadata, none provided") + if err := PrepareMetadata(&vmConfig.CloudInit, vmConfig.Name); err != nil { + return &VM{}, fmt.Errorf("failed to prepare cloud-init metadata: %s", err) + } + log.Infof("newVM:updated CloudInit data after prepare") + log.Infof("newVM: network-config: %s", vmConfig.CloudInit.NetworkConfig) + log.Infof("newVM: user-data: %s", vmConfig.CloudInit.UserData) + log.Infof("newVM: meta-data: %s", vmConfig.CloudInit.MetaData) + } + + // render CloudConfig to VM state dir if needed + seedDir := filepath.Join(runDir, "seed") + if !PathExists(seedDir) { + if err := CreateLocalDataSource(vmConfig.CloudInit, seedDir); err != nil { + // FIXME: if this create fails we might want to keep the image around? + os.RemoveAll(seedDir) + return &VM{}, fmt.Errorf("failed to create cloud-init datasource: %s", err) + } + } + + // create a vvfat block device (id, directory, fslabel) + seedBlkID := fmt.Sprintf("%s-cloudcfg", vmConfig.Name) + seedBlockDev, err := NewVVFATBlockDev(seedBlkID, seedDir, NoCloudFSLabel) + if err != nil { + return &VM{}, fmt.Errorf("failed to create a vvfat block device: %s", err) + } + + // insert vvfat blkdev if not already present + found := false + for n := range qcfg.BlkDevices { + if qcfg.BlkDevices[n].ID == seedBlkID { + found = true + } + } + + if !found { + qcfg.BlkDevices = append(qcfg.BlkDevices, seedBlockDev) + } + } + cmdParams, err := qcli.ConfigureParams(qcfg, nil) if err != nil { return &VM{}, fmt.Errorf("Failed to generate new VM command parameters: %s", err) @@ -315,7 +365,7 @@ func (v *VM) runVM() error { v.Cmd.Stderr = &stderr err := v.Cmd.Start() if err != nil { - errCh <- fmt.Errorf("VM:%s failed with: %s", stderr.String()) + errCh <- fmt.Errorf("VM:%s failed with: %s", v.Name(), stderr.String()) return } @@ -472,7 +522,7 @@ func (v *VM) Start() error { log.Infof("VM:%s starting...", v.Name()) err := v.BackgroundRun() if err != nil { - log.Errorf("VM:%s failed to start VM:%s %s", v.Name(), err) + log.Errorf("VM:%s failed to start VM: %s", v.Name(), err) v.Stop(true) return err } @@ -520,7 +570,7 @@ func (v *VM) Stop(force bool) error { case <-time.After(timeout): log.Warnf("VM:%s timed out, killing via cancel context...", v.Name()) v.Cancel() - log.Warnf("VM:%s cancel() complete") + log.Warnf("VM:%s cancel() complete", v.Name()) } v.wg.Wait() } else {