feat: add support for configuring/injecting cloud-init into VMs

.github/workflows/build.yml

@@ -21,6 +21,9 @@ jobs:
           make
           mv bin/machine bin/machine-linux-amd64
           mv bin/machined bin/machined-linux-amd64
+      - name: Test machine unittests
+        run: |
+          make test
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -53,6 +56,7 @@ jobs:
             make
             mv bin/machine bin/machine-linux-arm64
             mv bin/machined bin/machined-linux-arm64
+            make test
       - name: Release
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')

Makefile

@@ -3,6 +3,9 @@ BINS := bin/machine bin/machined
 .PHONY: all clean
 all: $(BINS)
 
+.PHONY: test
+test: test-api
+
 clean:
 	rm -f -v $(BINS)
 
@@ -11,3 +14,6 @@ bin/machine: cmd/machine/cmd/*.go pkg/*/*.go
 
 bin/machined: cmd/machined/cmd/*.go pkg/*/*.go
 	go build -o $@ cmd/machined/cmd/*.go
+
+test-api:
+	go test pkg/api/*.go

README.md

@@ -132,3 +132,8 @@ $ bin/machine start vm1
 200 OK
 $ bin/machine gui vm1
 ```
+
+
+## Examples
+
+See [doc/examples](doc/examples/) for other example VM definitions.

doc/examples/vm-with-cloud-init.yaml

@@ -0,0 +1,25 @@
+name: f40-vm1
+type: kvm
+ephemeral: false
+description: Fedora 40 Beta with UKI
+config:
+  name: f40-vm1
+  uefi: true
+  tpm: true
+  gui: false
+  tpm-version: 2.0
+  secure-boot: false
+  uefi-code: /usr/share/OVMF/OVMF_CODE.fd
+  disks:
+      - file: import/Fedora-Cloud-Base-UEFI-UKI.x86_64-40-1.10.qcow2
+        type: ssd
+        format: qcow2
+  cloud-init:
+      user-data: |
+        #cloud-config
+        password: <secret here>
+        chpasswd: { expire: False }
+        ssh_pwauth: True
+        ssh-authorized-keys:
+        - |
+          ssh-ed25519 xxxxx

go.mod

@@ -8,15 +8,17 @@ require (
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
 	github.com/gin-gonic/gin v1.8.1
 	github.com/go-resty/resty/v2 v2.7.0
+	github.com/google/uuid v1.3.0
 	github.com/lxc/lxd v0.0.0-20221130220346-2c77027b7a5e
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/msoap/byline v1.1.1
-	github.com/project-machine/qcli v0.2.1
+	github.com/pkg/errors v0.9.1
+	github.com/project-machine/qcli v0.3.1
 	github.com/rodaine/table v1.1.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.14.0
-	golang.org/x/sys v0.2.0
+	golang.org/x/sys v0.5.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -28,7 +30,6 @@ require (
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-playground/validator/v10 v10.10.0 // indirect
 	github.com/goccy/go-json v0.9.7 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect

go.sum

@@ -204,14 +204,15 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
 github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
 github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/project-machine/qcli v0.2.1 h1:rIRItjdkeBbD4NIxYyTkxCeJIolGHdniJ51Phfg2Ols=
-github.com/project-machine/qcli v0.2.1/go.mod h1:N7+8pGJWD/PvJOmzY6dmor4DYnG18bW/Mwa5Ful0GEs=
+github.com/project-machine/qcli v0.3.1 h1:OIiLUZa6acaFgtT/Ev//cehxblBSEtsqBNSNGVnyrc4=
+github.com/project-machine/qcli v0.3.1/go.mod h1:DEmDcRYrSL4s1DuYY7DxL3vEW6fMiVAQ6Ku7Fi1lOg8=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -404,8 +405,8 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=

pkg/api/cloudinit.go

@@ -0,0 +1,202 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package api
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/google/uuid"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	NoCloudFSLabel = "cidata"
+)
+
+/*
+type: kvm
+config:
+
+	name: slick-seal
+	...
+	cloud-init:
+	    user-data:|
+	        #cloud-config
+	        runcmd:
+	        - cat /etc/os-release
+	    network-config:|
+	        version: 2
+	        ethernets:
+	            nic0:
+	                match:
+	                    name: en*
+	                dhcp4: true
+	    meta-data:|
+	        instance-id: 08b2083d-2935-4d50-a442-d1da8920de20
+	        local-hostname: slick-seal
+*/
+
+type CloudInitConfig struct {
+	NetworkConfig string `yaml:"network-config"`
+	UserData      string `yaml:"user-data"`
+	MetaData      string `yaml:"meta-data"`
+}
+
+type MetaData struct {
+	InstanceId    string `yaml:"instance-id"`
+	LocalHostname string `yaml:"local-hostname"`
+}
+
+func HasCloudConfig(config CloudInitConfig) bool {
+
+	if config.MetaData != "" {
+		return true
+	}
+	if config.UserData != "" {
+		return true
+	}
+	if config.NetworkConfig != "" {
+		return true
+	}
+	return false
+}
+
+func PrepareMetadata(config *CloudInitConfig, hostname string) error {
+	// update MetaData with local-hostname and instance-id if not set
+
+	if config.MetaData != "" {
+		return fmt.Errorf("cloud-init config has existing metadata")
+	}
+
+	iid := uuid.New()
+
+	md := MetaData{
+		InstanceId:    iid.String(),
+		LocalHostname: hostname,
+	}
+
+	content, err := yaml.Marshal(&md)
+	if err != nil {
+		return fmt.Errorf("failed to marshal metadata: %s", err)
+	}
+
+	config.MetaData = string(content)
+
+	return nil
+}
+
+func RenderCloudInitConfig(config CloudInitConfig, outputPath string) error {
+
+	renderedFiles := 0
+	for _, d := range []struct {
+		confFile string
+		confData string
+	}{
+		{
+			confFile: "network-config",
+			confData: config.NetworkConfig,
+		},
+		{
+			confFile: "user-data",
+			confData: config.UserData,
+		},
+		{
+			confFile: "meta-data",
+			confData: config.MetaData,
+		},
+	} {
+		if len(d.confData) > 0 {
+			configFile := filepath.Join(outputPath, d.confFile)
+			tempFile, err := os.CreateTemp("", "tmp-cloudinit-")
+			if err != nil {
+				return fmt.Errorf("failed to create a temp file for writing cloud-init %s file: %s", d.confFile, err)
+			}
+			defer tempFile.Close()
+			defer os.Remove(tempFile.Name())
+			if err := os.WriteFile(tempFile.Name(), []byte(d.confData), 0666); err != nil {
+				return fmt.Errorf("failed to write cloud-init %s file %q: %s", d.confFile, tempFile.Name(), err)
+			}
+			if err := os.Rename(tempFile.Name(), configFile); err != nil {
+				return fmt.Errorf("failed to rename temp file %q to %q: %s", tempFile.Name(), configFile, err)
+			}
+			renderedFiles++
+		}
+	}
+	if renderedFiles == 0 {
+		return fmt.Errorf("failed to render any cloud-init config files; maybe empty cloud-init config?")
+	}
+	return nil
+}
+
+func verifyCloudInitConfig(cfg CloudInitConfig, contentsDir string) error {
+
+	// read the extracted directory and validate CloudInitConfig files
+	err := filepath.Walk(contentsDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			contents, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			log.Infof("verifyCloudInitCfg: path:%s name:%s contents:%s", path, info.Name(), contents)
+			switch info.Name() {
+			case "network-config":
+				if cfg.NetworkConfig != string(contents) {
+					return fmt.Errorf("network-config: expected contents %q, got %q", cfg.NetworkConfig, string(contents))
+				}
+			case "user-data":
+				if cfg.UserData != string(contents) {
+					return fmt.Errorf("user-data: expected contents %q, got %q", cfg.UserData, string(contents))
+				}
+			case "meta-data":
+				if cfg.MetaData != string(contents) {
+					return fmt.Errorf("meta-data: expected contents %q, got %q", cfg.MetaData, string(contents))
+				}
+			default:
+				return fmt.Errorf("Unexpected file %q in cloud-init rendered directory", info.Name())
+			}
+		} else {
+			if info.Name() != filepath.Base(path) {
+				return fmt.Errorf("Unexpected directory %q in cloud-init rendered directory", info.Name())
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func CreateLocalDataSource(cfg CloudInitConfig, directory string) error {
+
+	if err := EnsureDir(directory); err != nil {
+		return fmt.Errorf("failed to create cloud-init data source directory %q: %s", directory, err)
+	}
+
+	if err := RenderCloudInitConfig(cfg, directory); err != nil {
+		return fmt.Errorf("failed to render cloud-init config to directory %q: %s", directory, err)
+	}
+
+	if err := verifyCloudInitConfig(cfg, directory); err != nil {
+		return fmt.Errorf("failed to verify cloud-init config content in directory %q: %s", directory, err)
+	}
+
+	return nil
+}