From 1c5d33526f12034651a34aef55ec1239b74073e9 Mon Sep 17 00:00:00 2001 From: Valentin Flaux <38909103+vflaux@users.noreply.github.com> Date: Thu, 25 Jul 2024 14:28:10 +0200 Subject: [PATCH] fix containerd config generation --- pkg/oci/containerd.go | 68 ++++++++++++++++++++++++++++++++++++-- pkg/oci/containerd_test.go | 64 +++++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/pkg/oci/containerd.go b/pkg/oci/containerd.go index fb32a95a..7e38c401 100644 --- a/pkg/oci/containerd.go +++ b/pkg/oci/containerd.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "reflect" "strings" "github.com/containerd/containerd" @@ -21,6 +22,7 @@ import ( "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pelletier/go-toml/v2" + tomlu "github.com/pelletier/go-toml/v2/unstable" "github.com/spf13/afero" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -312,6 +314,13 @@ func createFilters(registries []url.URL) (string, string) { type hostFile struct { HostConfigs map[string]hostConfig `toml:"host"` Server string `toml:"server"` + + orderedHosts []string `toml:"-"` +} + +type hostFileOut struct { + HostConfigs any `toml:"host"` + Server string `toml:"server"` } type hostConfig struct { @@ -350,15 +359,48 @@ func AddMirrorConfiguration(ctx context.Context, fs afero.Fs, configPath string, if resolveTags { capabilities = append(capabilities, "resolve") } + + hostConfigType := reflect.TypeFor[hostConfig]() for _, registryURL := range registryURLs { hf, appending, err := getHostFile(fs, configPath, appendToBackup, registryURL) if err != nil { return err } + + // We make a struct to represent the hosts which allow us to control the order of the hosts in the output toml table + // Hosts order is important as containerd will try them in the order of apparition + // The order with a map would be unspecified while the go-toml lib guarantees the same order as the struct fields + hostConfigSize := len(registryURLs) + len(hf.HostConfigs) + hostConfigFields := make([]reflect.StructField, 0, hostConfigSize) + hostConfigValues := make([]hostConfig, 0, hostConfigSize) + addHost := func(host string, config hostConfig) { + hostConfigFields = append(hostConfigFields, reflect.StructField{ + Name: fmt.Sprintf("F%d", len(hostConfigFields)), + Type: hostConfigType, + Tag: reflect.StructTag(fmt.Sprintf("toml:\"%s\"", host)), + }) + hostConfigValues = append(hostConfigValues, config) + } for _, u := range mirrorURLs { - hf.HostConfigs[u.String()] = hostConfig{Capabilities: capabilities} + addHost(u.String(), hostConfig{Capabilities: capabilities}) + } + for _, h := range hf.orderedHosts { + addHost(h, hf.HostConfigs[h]) + } + + // Instantiate struct and assign values + hostsConfigs := reflect.New(reflect.StructOf(hostConfigFields)) + for i, v := range hostConfigValues { + f := hostsConfigs.Elem().Field(i) + f.Set(reflect.ValueOf(v)) } - b, err := toml.Marshal(&hf) + + out := &hostFileOut{ + Server: hf.Server, + } + reflect.ValueOf(&out.HostConfigs).Elem().Set(hostsConfigs) + + b, err := toml.Marshal(out) if err != nil { return err } @@ -462,6 +504,7 @@ func getHostFile(fs afero.Fs, configPath string, appendToBackup bool, registryUR if err != nil { return hostFile{}, false, err } + hf.orderedHosts = getOrderedHosts(b) return hf, true, nil } } @@ -475,3 +518,24 @@ func getHostFile(fs afero.Fs, configPath string, appendToBackup bool, registryUR } return hf, false, nil } + +// getOrderedHosts parses the given byte slice as TOML data and returns a slice of +// strings containing the ordered hosts found in the TOML data. +func getOrderedHosts(b []byte) []string { + p := tomlu.Parser{} + p.Reset(b) + orderedHosts := []string{} + for p.NextExpression() { + e := p.Expression() + if e.Kind != tomlu.Table { + continue + } + ki := e.Key() + // check if the expression is a host ("[host.'xxx']") + if ki.Next() && string(ki.Node().Data) == "host" && + ki.Next() && ki.IsLast() { + orderedHosts = append(orderedHosts, string(ki.Node().Data)) + } + } + return orderedHosts +} diff --git a/pkg/oci/containerd_test.go b/pkg/oci/containerd_test.go index e1b60974..75f0e0a0 100644 --- a/pkg/oci/containerd_test.go +++ b/pkg/oci/containerd_test.go @@ -208,10 +208,10 @@ func TestMirrorConfiguration(t *testing.T) { appendToBackup bool }{ { - name: "multiple mirros", + name: "multiple mirrors", resolveTags: true, registries: stringListToUrlList(t, []string{"http://foo.bar:5000"}), - mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000", "http://127.0.0.1:5001"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000", "http://127.0.0.2:5001", "http://127.0.0.1:5001"}), expectedFiles: map[string]string{ "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' @@ -219,6 +219,9 @@ func TestMirrorConfiguration(t *testing.T) { [host.'http://127.0.0.1:5000'] capabilities = ['pull', 'resolve'] +[host.'http://127.0.0.2:5001'] +capabilities = ['pull', 'resolve'] + [host.'http://127.0.0.1:5001'] capabilities = ['pull', 'resolve'] `, @@ -360,6 +363,10 @@ client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] [host.'http://example.com:30021'] capabilities = ['pull', 'resolve'] client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] + +[host.'http://bar.com:30020'] +capabilities = ['pull', 'resolve'] +client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] `, }, expectedFiles: map[string]string{ @@ -373,6 +380,10 @@ client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] [host.'http://example.com:30021'] capabilities = ['pull', 'resolve'] client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] + +[host.'http://bar.com:30020'] +capabilities = ['pull', 'resolve'] +client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] `, "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' @@ -387,6 +398,10 @@ capabilities = ['pull', 'resolve'] [host.'http://example.com:30021'] client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] capabilities = ['pull', 'resolve'] + +[host.'http://bar.com:30020'] +client = ['/etc/certs/xxx/client.cert', '/etc/certs/xxx/client.key'] +capabilities = ['pull', 'resolve'] `, "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' @@ -466,3 +481,48 @@ func stringListToUrlList(t *testing.T, list []string) []url.URL { } return urls } + +func TestGetOrderedHosts(t *testing.T) { + type args struct { + b []byte + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "empty file", + args: args{b: []byte{}}, + want: []string{}, + }, + { + name: "ordered hosts", + args: args{b: []byte(`server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.4:5000'] +capabilities = ['pull', 'resolve'] + +[host.'http://127.0.0.3:5000'] +capabilities = ['pull', 'resolve'] + +[host.'http://127.0.0.2:5000'] +capabilities = ['pull', 'resolve'] + +[host.'http://127.0.0.5:5000'] +capabilities = ['pull', 'resolve'] + +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`)}, + want: []string{"http://127.0.0.4:5000", "http://127.0.0.3:5000", "http://127.0.0.2:5000", "http://127.0.0.5:5000", "http://127.0.0.1:5000"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getOrderedHosts(tt.args.b) + require.Equal(t, tt.want, got) + }) + } +}