test(e2e): command-surface release gate (db/s3/expose/localstack)

.github/workflows/nightly.yml

@@ -48,6 +48,29 @@ jobs:
       - name: nightly gate (fmt-check + vet + cross-build + test-race + determinism)
         run: make nightly
 
+  # Heavy cloud-engine command e2e (localstack + the `aws` shim, plus the full
+  # db/s3/expose command surface). It pulls large images (localstack), so it runs
+  # here as a nightly RELEASE GATE rather than on every PR — the per-PR CI already
+  # runs the lighter postgres+minio+redis command e2e (DEVSTACK_E2E=1). This is the
+  # "every command actually works against a live stack" gate.
+  e2e-cloud:
+    if: github.repository == 'open-source-cloud/devstack'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+          cache: true
+      - name: docker available
+        run: docker version
+      - name: cloud command e2e (localstack health gate + aws shim + db/s3/expose)
+        env:
+          DEVSTACK_E2E: "1"
+          DEVSTACK_E2E_CLOUD: "1"
+        run: go test -tags=e2e ./tests/e2e/... -run 'Commands|LocalStack' -count=1 -v
+
   # Optional rolling pre-release. Default OFF: enable by setting the repo variable
   #   gh variable set NIGHTLY_PRERELEASE --body true
   # Produces goreleaser SNAPSHOT artifacts (no real version, no git tag) and uploads

internal/cli/expose.go

@@ -0,0 +1,116 @@
+package cli
+
+import (
+	"fmt"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+
+	"github.com/open-source-cloud/devstack/internal/orchestrate"
+)
+
+// newSharedExposeCmd wires `shared expose [services...]` — publish the shared
+// engines on stable 127.0.0.1 host ports so GUI clients (DataGrip, a Redis/S3
+// browser, the RabbitMQ UI) can connect. Opt-in and loopback-only; it never
+// touches the deterministic generated compose (an up-time overlay). `--off`
+// removes the publish and returns the stack to DNS-only.
+func newSharedExposeCmd(g *GlobalOpts) *cobra.Command {
+	var off bool
+	cmd := &cobra.Command{
+		Use:   "expose [services...]",
+		Short: "Publish shared services on stable 127.0.0.1 ports for local GUI clients",
+		Long: "Publish the shared engines on stable 127.0.0.1 host ports so host tools and GUI\n" +
+			"clients (DataGrip, TablePlus, a Redis/S3 browser, the RabbitMQ management UI)\n" +
+			"can reach them. Ports are ledger-allocated (stable across runs) and loopback-only.\n" +
+			"With no arguments, every exposable shared service is published; name services to\n" +
+			"scope it. `--off` removes the publish. The persist survives up/down.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			d, closeFn, err := buildUpDeps(cmd)
+			if err != nil {
+				return err
+			}
+			defer closeFn()
+			if off {
+				if err := orchestrate.UnexposeShared(cmd.Context(), d); err != nil {
+					return err
+				}
+				if g.JSON {
+					return writeJSON(cmd, map[string]any{"exposed": []any{}})
+				}
+				if !g.Quiet {
+					fmt.Fprintln(cmd.OutOrStdout(), "shared services are DNS-only again (host ports removed)")
+				}
+				return nil
+			}
+			ports, err := orchestrate.ExposeShared(cmd.Context(), d, args)
+			if err != nil {
+				return err
+			}
+			return renderExposed(cmd, g, ports)
+		},
+	}
+	cmd.Flags().BoolVar(&off, "off", false, "remove the host-port publish (back to DNS-only)")
+	return cmd
+}
+
+// newSharedPortsCmd wires `shared ports` — the read-only projection of the
+// currently-published host ports + connection strings (lock-free snapshot).
+func newSharedPortsCmd(g *GlobalOpts) *cobra.Command {
+	return &cobra.Command{
+		Use:   "ports",
+		Short: "Show the published 127.0.0.1 host ports for shared services (and connection strings)",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			d, closeFn, err := buildUpDeps(cmd)
+			if err != nil {
+				return err
+			}
+			defer closeFn()
+			ports, err := orchestrate.ExposedStatus(cmd.Context(), d)
+			if err != nil {
+				return err
+			}
+			if len(ports) == 0 && !g.JSON {
+				fmt.Fprintln(cmd.OutOrStdout(), "no shared services exposed — run `devstack shared expose`")
+				return nil
+			}
+			return renderExposed(cmd, g, ports)
+		},
+	}
+}
+
+// renderExposed prints the exposed-port projection as JSON or an aligned table.
+func renderExposed(cmd *cobra.Command, g *GlobalOpts, ports []orchestrate.ExposedPort) error {
+	if g.JSON {
+		return writeJSON(cmd, map[string]any{"exposed": ports})
+	}
+	if g.Quiet {
+		for _, p := range ports {
+			if p.URL != "" {
+				fmt.Fprintln(cmd.OutOrStdout(), p.URL)
+			}
+		}
+		return nil
+	}
+	tw := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "SERVICE\tPORT\tADDRESS\tCONNECT")
+	for _, p := range ports {
+		label := p.Alias
+		if !p.Primary {
+			label = p.Alias + " (" + p.Label + ")"
+		}
+		fmt.Fprintf(tw, "%s\t%s\t127.0.0.1:%d\t%s\n", label, p.Label, p.Port, p.URL)
+	}
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+	// A one-line reminder that per-project Postgres DBs use their own dev creds.
+	for _, p := range ports {
+		if p.Engine == "postgres" && p.Primary {
+			fmt.Fprintf(cmd.OutOrStdout(),
+				"\nper-project database: postgres://<project>:<project>@127.0.0.1:%d/<project>?sslmode=disable\n", p.Port)
+			break
+		}
+	}
+	return nil
+}

internal/cli/expose_test.go

@@ -0,0 +1,76 @@
+package cli
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+
+	"github.com/open-source-cloud/devstack/internal/orchestrate"
+)
+
+func exposeFixture() []orchestrate.ExposedPort {
+	return []orchestrate.ExposedPort{
+		{Instance: "postgres", Engine: "postgres", Alias: "shared-postgres", Label: "postgres", Host: "127.0.0.1", Port: 55432, Container: 5432, Primary: true, URL: "postgres://devstack:devstack@127.0.0.1:55432/postgres?sslmode=disable"},
+		{Instance: "minio", Engine: "minio", Alias: "shared-minio", Label: "console", Host: "127.0.0.1", Port: 59001, Container: 9001, Primary: false, URL: "http://127.0.0.1:59001"},
+	}
+}
+
+func TestRenderExposed_Table(t *testing.T) {
+	var buf bytes.Buffer
+	cmd := &cobra.Command{}
+	cmd.SetOut(&buf)
+	if err := renderExposed(cmd, &GlobalOpts{}, exposeFixture()); err != nil {
+		t.Fatal(err)
+	}
+	out := buf.String()
+	for _, want := range []string{"shared-postgres", "55432", "shared-minio (console)", "59001", "per-project database:"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("table missing %q:\n%s", want, out)
+		}
+	}
+}
+
+func TestRenderExposed_JSON(t *testing.T) {
+	var buf bytes.Buffer
+	cmd := &cobra.Command{}
+	cmd.SetOut(&buf)
+	if err := renderExposed(cmd, &GlobalOpts{JSON: true}, exposeFixture()); err != nil {
+		t.Fatal(err)
+	}
+	out := buf.String()
+	if !strings.Contains(out, "\"exposed\"") || !strings.Contains(out, "\"port\": 55432") {
+		t.Errorf("json missing fields:\n%s", out)
+	}
+}
+
+func TestRenderExposed_Quiet(t *testing.T) {
+	var buf bytes.Buffer
+	cmd := &cobra.Command{}
+	cmd.SetOut(&buf)
+	if err := renderExposed(cmd, &GlobalOpts{Quiet: true}, exposeFixture()); err != nil {
+		t.Fatal(err)
+	}
+	out := strings.TrimSpace(buf.String())
+	// Quiet emits only the connection URLs, one per line.
+	lines := strings.Split(out, "\n")
+	if len(lines) != 2 || !strings.HasPrefix(lines[0], "postgres://") {
+		t.Errorf("quiet should print only URLs, got:\n%s", out)
+	}
+}
+
+// TestSharedExposeCommandsRegistered guards that `shared expose` and
+// `shared ports` are wired into the shared command tree.
+func TestSharedExposeCommandsRegistered(t *testing.T) {
+	sh := newSharedCmd(&GlobalOpts{})
+	have := map[string]bool{}
+	for _, c := range sh.Commands() {
+		have[c.Name()] = true
+	}
+	for _, want := range []string{"expose", "ports", "status", "gc", "doctor"} {
+		if !have[want] {
+			t.Errorf("shared subcommand %q not registered", want)
+		}
+	}
+}

internal/cli/shared.go

@@ -27,6 +27,8 @@ func newSharedCmd(g *GlobalOpts) *cobra.Command {
 		newSharedStatusCmd(g),
 		newSharedGcCmd(g),
 		newSharedDoctorCmd(g),
+		newSharedExposeCmd(g),
+		newSharedPortsCmd(g),
 	)
 	return cmd
 }

internal/generate/cloud_engines_test.go

@@ -50,6 +50,31 @@ func TestCloudEngineTemplatesLint(t *testing.T) {
 	}
 }
 
+// TestLocalStackHealthGatesOnAvailable guards the fix for the "shared-localstack
+// unhealthy after 1 attempt" bug. LocalStack 3.x reports each configured SERVICE
+// as "available" on startup — a service only flips to "running" after its first
+// request. A healthcheck that greps solely for "running" therefore NEVER passes
+// (nothing is running until traffic arrives), the container stays unhealthy, and
+// the up saga aborts. The gate must accept "available".
+func TestLocalStackHealthGatesOnAvailable(t *testing.T) {
+	src := template.NewFSSource(templates.FS)
+	res, err := template.Resolve(src, "localstack", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compose, err := LintResolved("localstack", res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(compose)
+	if !strings.Contains(s, "available") {
+		t.Errorf("localstack healthcheck must accept the \"available\" state, not gate solely on \"running\":\n%s", s)
+	}
+	if strings.Contains(s, "grep -q running") {
+		t.Error("localstack healthcheck still greps solely for \"running\" — the deadlock bug")
+	}
+}
+
 // TestRabbitMQSecretIsValueless asserts RABBITMQ_DEFAULT_PASS is emitted as a
 // valueless env key (no plaintext) — the §7.5 secret coupling for broker creds.
 func TestRabbitMQSecretIsValueless(t *testing.T) {

internal/orchestrate/connect_retry.go

@@ -0,0 +1,107 @@
+package orchestrate
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/open-source-cloud/devstack/internal/provision"
+)
+
+// This file hardens the host-side Postgres admin connection against the
+// readiness race that surfaced as `db create` failing with "connect to shared
+// postgres on 127.0.0.1:<port>: read: connection reset by peer".
+//
+// Why the race exists: the imperative resource path (and the up provision phase)
+// publishes the shared engine's host port via an up-time compose overlay, then
+// `docker compose up -d <inst>` applies it. Adding a published port RECREATES the
+// container, so Postgres restarts; for a second or two afterwards Docker's
+// userland proxy accepts the TCP connection on 127.0.0.1:<port> but the backend
+// isn't listening yet, so it RSTs the handshake ("connection reset by peer",
+// "failed to receive message", EOF). A single immediate connect loses the race.
+//
+// The fix mirrors how any client should treat a just-(re)started server: retry
+// the connect with backoff for a bounded window. Idempotent and safe — a healthy
+// server connects on the first try, so this only ever adds latency on the race.
+
+const (
+	// connectRetryBudget bounds how long we retry a transient connect before
+	// giving up and surfacing the real error (Postgres genuinely down / wrong
+	// creds fail fast because those errors are not transient).
+	connectRetryBudget = 30 * time.Second
+	// connectRetryStart is the initial backoff; it doubles up to connectRetryMax.
+	connectRetryStart = 200 * time.Millisecond
+	connectRetryMax   = 2 * time.Second
+)
+
+// transientConnErr reports whether a Postgres connect error is the engine still
+// coming up after a port-overlay recreate (retry) rather than a permanent
+// failure like bad credentials or an unknown database (fail fast).
+func transientConnErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	s := strings.ToLower(err.Error())
+	for _, m := range []string{
+		"connection reset by peer",
+		"connection refused",
+		"failed to receive message",
+		"the database system is starting up",
+		"broken pipe",
+		"unexpected eof",
+		"eof",
+		"i/o timeout",
+		"no route to host",
+		"server closed the connection unexpectedly",
+	} {
+		if strings.Contains(s, m) {
+			return true
+		}
+	}
+	return false
+}
+
+// sleepFn is indirected so tests can drive the backoff without real time.
+var sleepFn = func(ctx context.Context, d time.Duration) error {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-t.C:
+		return nil
+	}
+}
+
+// nowFn is indirected for tests.
+var nowFn = time.Now
+
+// retryingPgConnect wraps a PgConnector so a transient connect error (the engine
+// just restarted to bind its host port) is retried with capped backoff for
+// connectRetryBudget. A nil connector passes through nil (the default connector
+// is substituted downstream). Non-transient errors and a cancelled context
+// return immediately.
+func retryingPgConnect(connect PgConnector) PgConnector {
+	if connect == nil {
+		return nil
+	}
+	return func(ctx context.Context, dsn string) (provision.Conn, func() error, error) {
+		deadline := nowFn().Add(connectRetryBudget)
+		backoff := connectRetryStart
+		for {
+			conn, closeFn, err := connect(ctx, dsn)
+			if err == nil {
+				return conn, closeFn, nil
+			}
+			if !transientConnErr(err) || !nowFn().Before(deadline) || ctx.Err() != nil {
+				return nil, nil, err
+			}
+			if serr := sleepFn(ctx, backoff); serr != nil {
+				return nil, nil, err
+			}
+			if backoff < connectRetryMax {
+				backoff *= 2
+			}
+		}
+	}
+}