diff --git a/internal/cli/db.go b/internal/cli/db.go index d89e9ea..85ba2ad 100644 --- a/internal/cli/db.go +++ b/internal/cli/db.go @@ -6,6 +6,8 @@ import ( "github.com/spf13/cobra" + "github.com/open-source-cloud/devstack/internal/db" + "github.com/open-source-cloud/devstack/internal/docker" "github.com/open-source-cloud/devstack/internal/orchestrate" "github.com/open-source-cloud/devstack/internal/resource" "github.com/open-source-cloud/devstack/internal/state" @@ -14,8 +16,9 @@ import ( // newDbCmd wires the `devstack db` group (spec 29 §databases): tenant-scoped // Postgres database + role/grant verbs on the shared engine. create/user/grant/ // drop/gc mirror the up-saga provision flow (lock → overlay → provisioner → -// ledger → event) via internal/orchestrate; list is a lock-free ledger read. This -// graduates the reserved `db` stub. snapshot/restore/reset/pull stay v2 stubs. +// ledger → event) via internal/orchestrate; list is a lock-free ledger read. +// snapshot/restore (+ snapshot ls) graduate the spec-15 data-lifecycle verbs; +// reset/pull stay v2 stubs. func newDbCmd(g *GlobalOpts) *cobra.Command { cmd := &cobra.Command{ Use: "db", @@ -28,15 +31,159 @@ func newDbCmd(g *GlobalOpts) *cobra.Command { newDbListCmd(g), newDbDropCmd(g), newDbGcCmd(g), - // v2 data-lifecycle verbs (spec 15) reserved as stubs. - stub("snapshot", "Snapshot a project's database", "v2 (spec 15)"), - stub("restore", "Restore a project's database from a snapshot", "v2 (spec 15)"), + // spec-15 data-lifecycle verbs. + newDbSnapshotCmd(g), + newDbRestoreCmd(g), + // remaining spec-15 verbs (reset/pull) reserved as stubs. stub("reset", "Drop and re-provision a project's database", "v2 (spec 15)"), stub("pull", "Pull a database snapshot from a shared store", "v2 (spec 15)"), ) return cmd } +// defaultPgDumper is the real pg_dump/pg_restore/psql client, shelled behind the +// docker exec runner (the release binary stays CGO-free — the tools are external). +func defaultPgDumper() db.Dumper { return db.PgDumper{Runner: docker.ExecRunner{}} } + +// newDbSnapshotCmd wires `db snapshot [name]` (capture) with the `ls` subcommand +// (list). A snapshot dumps ONLY the project's tenant database on the shared +// Postgres to ~/.devstack/snapshots// and records a ledger row (spec 15). +func newDbSnapshotCmd(g *GlobalOpts) *cobra.Command { + var project, database, instance string + cmd := &cobra.Command{ + Use: "snapshot [name]", + Short: "Capture a project's tenant database to the snapshot store", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + d, closeFn, err := buildUpDeps(cmd) + if err != nil { + return err + } + defer closeFn() + dumper := defaultPgDumper() + if err := dumper.Preflight(cmd.Context()); err != nil { + return err + } + var name string + if len(args) == 1 { + name = args[0] + } + meta, err := orchestrate.Snapshot(cmd.Context(), d, dumper, orchestrate.SnapshotOptions{ + Project: project, Database: database, Instance: instance, Name: name, + }) + if err != nil { + return err + } + if g.JSON { + return writeJSON(cmd, meta) + } + if !g.Quiet { + fmt.Fprintf(cmd.OutOrStdout(), "captured snapshot %q of %s (%d bytes)\n%s\n", meta.Name, meta.Database, meta.Size, meta.Path) + } + return nil + }, + } + cmd.Flags().StringVar(&project, "project", "", "owner project (default: the workspace's single/first project)") + cmd.Flags().StringVar(&database, "db", "", "physical tenant database (default: the project's own database)") + cmd.Flags().StringVar(&instance, "instance", "", "shared postgres instance (default: the first postgres instance)") + cmd.AddCommand(newDbSnapshotLsCmd(g)) + return cmd +} + +func newDbSnapshotLsCmd(g *GlobalOpts) *cobra.Command { + var project string + cmd := &cobra.Command{ + Use: "ls", + Short: "List captured snapshots (lock-free)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + d, closeFn, err := buildUpDeps(cmd) + if err != nil { + return err + } + defer closeFn() + snaps, err := orchestrate.ListSnapshots(d, project) + if err != nil { + return err + } + if g.JSON { + return writeJSON(cmd, map[string]any{"snapshots": snaps}) + } + w := cmd.OutOrStdout() + if len(snaps) == 0 { + fmt.Fprintln(w, "no snapshots") + return nil + } + for _, s := range snaps { + fmt.Fprintf(w, "%-24s %-12s %10d %s %s\n", s.Name, s.Database, s.Size, shortDigest(s.Digest), s.CreatedAt) + } + return nil + }, + } + cmd.Flags().StringVar(&project, "project", "", "only this project's snapshots") + return cmd +} + +func newDbRestoreCmd(g *GlobalOpts) *cobra.Command { + var project, database, instance string + var force, yes bool + cmd := &cobra.Command{ + Use: "restore ", + Short: "Restore a project's tenant database from a snapshot (destructive)", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if g.JSON && !yes { + return fmt.Errorf("refusing to restore without --yes for --json/non-interactive use") + } + d, closeFn, err := buildUpDeps(cmd) + if err != nil { + return err + } + defer closeFn() + dumper := defaultPgDumper() + if err := dumper.Preflight(cmd.Context()); err != nil { + return err + } + if !yes { + if !confirm(cmd, fmt.Sprintf("This REPLACES the tenant database from snapshot %q (current data destroyed). Type 'yes' to continue: ", args[0])) { + fmt.Fprintln(cmd.OutOrStdout(), "aborted") + return nil + } + } + meta, err := orchestrate.Restore(cmd.Context(), d, dumper, orchestrate.RestoreOptions{ + Project: project, Database: database, Instance: instance, Name: args[0], Force: force, + }) + if err != nil { + return err + } + if g.JSON { + return writeJSON(cmd, meta) + } + if !g.Quiet { + fmt.Fprintf(cmd.OutOrStdout(), "restored %s from snapshot %q (digest %s)\n", meta.Database, meta.Name, shortDigest(meta.Digest)) + } + return nil + }, + } + cmd.Flags().StringVar(&project, "project", "", "owner project (default: the workspace's single/first project)") + cmd.Flags().StringVar(&database, "db", "", "physical tenant database (default: the project's own database)") + cmd.Flags().StringVar(&instance, "instance", "", "shared postgres instance (default: the first postgres instance)") + cmd.Flags().BoolVar(&force, "force", false, "replay over a non-empty database (overwrite existing data)") + cmd.Flags().BoolVar(&yes, "yes", false, "skip the confirmation prompt (required for --json)") + return cmd +} + +// shortDigest is a display helper: the first 12 hex chars of a sha256, or "-". +func shortDigest(d string) string { + if len(d) >= 12 { + return d[:12] + } + if d == "" { + return "-" + } + return d +} + // sanitizePg maps a name to a safe Postgres identifier (hyphens → underscores), // matching provision.pgIdent so ledger + physical names line up. func sanitizePg(s string) string { return strings.ReplaceAll(s, "-", "_") } diff --git a/internal/cli/db_snapshot_test.go b/internal/cli/db_snapshot_test.go new file mode 100644 index 0000000..90d748f --- /dev/null +++ b/internal/cli/db_snapshot_test.go @@ -0,0 +1,49 @@ +package cli + +import "testing" + +func TestDbSnapshotCommandsRegistered(t *testing.T) { + root := NewRootCmd(Options{}) + for _, path := range [][]string{ + {"db", "snapshot"}, {"db", "snapshot", "ls"}, {"db", "restore"}, + } { + c, _, err := root.Find(path) + if err != nil || c.RunE == nil { + t.Fatalf("db %v not registered as a real command: %v", path, err) + } + } +} + +func TestDbSnapshotFlags(t *testing.T) { + root := NewRootCmd(Options{}) + snap, _, err := root.Find([]string{"db", "snapshot"}) + if err != nil { + t.Fatal(err) + } + for _, f := range []string{"project", "db", "instance"} { + if snap.Flags().Lookup(f) == nil { + t.Errorf("db snapshot missing --%s", f) + } + } + restore, _, err := root.Find([]string{"db", "restore"}) + if err != nil { + t.Fatal(err) + } + for _, f := range []string{"project", "db", "instance", "force", "yes"} { + if restore.Flags().Lookup(f) == nil { + t.Errorf("db restore missing --%s", f) + } + } +} + +func TestShortDigest(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"", "-"}, + {"abc", "abc"}, + {"0123456789abcdef", "0123456789ab"}, + } { + if got := shortDigest(tc.in); got != tc.want { + t.Errorf("shortDigest(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/db/pg.go b/internal/db/pg.go new file mode 100644 index 0000000..197f255 --- /dev/null +++ b/internal/db/pg.go @@ -0,0 +1,155 @@ +// Package db is the data-lifecycle seam for the shared engines (spec 15): it +// captures and replays a single project's tenant namespace on the warm shared +// Postgres via the engine's own external client tooling (pg_dump / pg_restore / +// psql). The tools are shelled behind the Dumper interface — exactly the +// docker/git wrapping discipline — so the release binary stays a pure-Go, +// CGO-free static binary and every risky external tool gets an internal/ seam +// plus a mock. Only the pg dumper exists in this milestone; redis/minio dumpers +// slot in behind the same interface (Full scope). +// +// The dumper never touches the shared container via the SDK: Compose owns +// containers, the tools run as HOST binaries against a ledger-allocated, +// 127.0.0.1-only host-port overlay (the same reachability path the provision +// phase uses). The password is passed via PGPASSWORD in the process env, never +// on the argv (so it does not leak into `ps`) — the same secret-handling posture +// as the rest of devstack (§7.5). +package db + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// Runner shells an external command. Its shape matches docker.Runner so the real +// docker.ExecRunner satisfies it directly, and tests inject a recording fake. +type Runner interface { + Run(ctx context.Context, env []string, dir, name string, args ...string) error + Output(ctx context.Context, env []string, dir, name string, args ...string) ([]byte, error) +} + +// ConnInfo is a host-reachable admin endpoint for one tenant database. Host/Port +// come from the ledger-allocated 127.0.0.1 overlay; User/Password are the shared +// instance's admin credentials; Database is the per-project tenant db. +type ConnInfo struct { + Host string + Port int + User string + Password string + Database string +} + +// Dumper captures and replays a single database behind an external client tool. +// Snapshot writes a content-addressable dump to outPath; Restore replays inPath +// into the (recreated/clean) database; IsEmpty reports whether the tenant has any +// user tables (the restore-over-non-empty guard); Preflight checks the tool is +// present and version-compatible. +type Dumper interface { + Preflight(ctx context.Context) error + Snapshot(ctx context.Context, conn ConnInfo, outPath string) error + Restore(ctx context.Context, conn ConnInfo, inPath string) error + IsEmpty(ctx context.Context, conn ConnInfo) (bool, error) +} + +// ErrToolMissing is returned by Preflight when the required client binary is not +// on PATH. It carries a one-line remediation (ARCHITECTURE §7.6). +type ErrToolMissing struct { + Tool string + Remediation string +} + +func (e *ErrToolMissing) Error() string { + return fmt.Sprintf("%s not found: %s", e.Tool, e.Remediation) +} + +// PgDumper shells pg_dump / pg_restore / psql. The client major must be ≥ the +// server major to restore reliably (spec 15); this milestone stores no version +// gate but the seam is here. Runner is injectable (nil → the real exec runner is +// supplied by the caller); LookPath is injectable so Preflight is unit-testable. +type PgDumper struct { + Runner Runner + LookPath func(string) (string, error) // nil → exec.LookPath +} + +// pgClientTools are the external binaries the pg dumper needs on PATH. +var pgClientTools = []string{"pg_dump", "pg_restore", "psql"} + +// Preflight verifies the PostgreSQL client tools are installed. Absence degrades +// the db verbs only (never blocks up), consistent with the mkcert/cloudflared +// external-binary posture (DECISIONS D11/D12). +func (p PgDumper) Preflight(_ context.Context) error { + look := p.LookPath + if look == nil { + look = exec.LookPath + } + for _, tool := range pgClientTools { + if _, err := look(tool); err != nil { + return &ErrToolMissing{ + Tool: tool, + Remediation: "install the PostgreSQL client tools (e.g. `apt install postgresql-client`, `brew install libpq`, or `dnf install postgresql`) so `" + tool + "` is on PATH", + } + } + } + return nil +} + +// connFlags builds the shared -h/-p/-U/-d connection flags. The password is NOT +// here — it rides PGPASSWORD in the env (pgEnv). +func connFlags(c ConnInfo) []string { + return []string{"-h", c.Host, "-p", strconv.Itoa(c.Port), "-U", c.User, "-d", c.Database} +} + +// pgEnv passes the password out-of-band so it never lands on the argv. +func pgEnv(c ConnInfo) []string { return []string{"PGPASSWORD=" + c.Password} } + +// Snapshot dumps the tenant database to outPath in the custom (compressed, +// selectively-restorable) format, owner-stripped so it replays into a +// freshly-recreated role. pg_dump opens a REPEATABLE READ snapshot, so it is +// consistent against a busy tenant WITHOUT stopping the shared server (spec 15). +func (p PgDumper) Snapshot(ctx context.Context, conn ConnInfo, outPath string) error { + args := append(connFlags(conn), "--format=custom", "--no-owner", "--no-privileges", "--file", outPath) + if err := p.Runner.Run(ctx, pgEnv(conn), "", "pg_dump", args...); err != nil { + return fmt.Errorf("pg_dump %s: %w", conn.Database, err) + } + return nil +} + +// Restore replays inPath into the tenant database, dropping conflicting objects +// first (--clean --if-exists) and ignoring dump ownership (--no-owner). The +// caller must have terminated live backends + recreated a clean database (the +// tenant scope) before calling; the drop/recreate SQL is provisioning's guarded +// pgx path (spec 15). --exit-on-error so a partial restore fails loudly. +func (p PgDumper) Restore(ctx context.Context, conn ConnInfo, inPath string) error { + args := append(connFlags(conn), "--clean", "--if-exists", "--no-owner", "--no-privileges", "--exit-on-error", inPath) + if err := p.Runner.Run(ctx, pgEnv(conn), "", "pg_restore", args...); err != nil { + return fmt.Errorf("pg_restore %s: %w", conn.Database, err) + } + return nil +} + +// emptyCountSQL counts user tables (excluding the system schemas) so a restore +// can refuse to clobber a tenant that already has data unless --force. +const emptyCountSQL = `SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema')` + +// IsEmpty reports whether the tenant database has no user tables. +func (p PgDumper) IsEmpty(ctx context.Context, conn ConnInfo) (bool, error) { + args := append(connFlags(conn), "-tAX", "-c", emptyCountSQL) + out, err := p.Runner.Output(ctx, pgEnv(conn), "", "psql", args...) + if err != nil { + return false, fmt.Errorf("psql count tables in %s: %w", conn.Database, err) + } + n, perr := strconv.Atoi(strings.TrimSpace(string(out))) + if perr != nil { + return false, fmt.Errorf("parse table count %q: %w", strings.TrimSpace(string(out)), perr) + } + return n == 0, nil +} + +// IsToolMissing reports whether err is (or wraps) an ErrToolMissing. +func IsToolMissing(err error) bool { + var e *ErrToolMissing + return errors.As(err, &e) +} diff --git a/internal/db/pg_test.go b/internal/db/pg_test.go new file mode 100644 index 0000000..cd39fa3 --- /dev/null +++ b/internal/db/pg_test.go @@ -0,0 +1,111 @@ +package db + +import ( + "context" + "errors" + "strings" + "testing" +) + +// recRunner records the argv + env of every shelled tool and returns a canned +// Output for psql. +type recRunner struct { + cmds [][]string + envs [][]string + output []byte +} + +func (r *recRunner) Run(_ context.Context, env []string, _, name string, args ...string) error { + r.cmds = append(r.cmds, append([]string{name}, args...)) + r.envs = append(r.envs, env) + return nil +} +func (r *recRunner) Output(_ context.Context, env []string, _, name string, args ...string) ([]byte, error) { + r.cmds = append(r.cmds, append([]string{name}, args...)) + r.envs = append(r.envs, env) + return r.output, nil +} + +func conn() ConnInfo { + return ConnInfo{Host: "127.0.0.1", Port: 45432, User: "devstack", Password: "s3cr3t", Database: "app"} +} + +func TestPgDumperSnapshotArgv(t *testing.T) { + r := &recRunner{} + p := PgDumper{Runner: r} + if err := p.Snapshot(context.Background(), conn(), "/tmp/app.dump"); err != nil { + t.Fatalf("Snapshot: %v", err) + } + argv := strings.Join(r.cmds[0], " ") + for _, want := range []string{"pg_dump", "-h 127.0.0.1", "-p 45432", "-U devstack", "-d app", "--format=custom", "--no-owner", "--file /tmp/app.dump"} { + if !strings.Contains(argv, want) { + t.Errorf("Snapshot argv missing %q: %s", want, argv) + } + } + // Password only in the env, never on argv. + if strings.Contains(argv, "s3cr3t") { + t.Errorf("password leaked into argv: %s", argv) + } + if got := strings.Join(r.envs[0], " "); got != "PGPASSWORD=s3cr3t" { + t.Errorf("env = %q, want PGPASSWORD=s3cr3t", got) + } +} + +func TestPgDumperRestoreArgv(t *testing.T) { + r := &recRunner{} + p := PgDumper{Runner: r} + if err := p.Restore(context.Background(), conn(), "/tmp/app.dump"); err != nil { + t.Fatalf("Restore: %v", err) + } + argv := strings.Join(r.cmds[0], " ") + for _, want := range []string{"pg_restore", "-d app", "--clean", "--if-exists", "--no-owner", "/tmp/app.dump"} { + if !strings.Contains(argv, want) { + t.Errorf("Restore argv missing %q: %s", want, argv) + } + } +} + +func TestPgDumperIsEmpty(t *testing.T) { + for _, tc := range []struct { + out string + want bool + }{ + {"0\n", true}, + {"7\n", false}, + {" 0 ", true}, + } { + r := &recRunner{output: []byte(tc.out)} + p := PgDumper{Runner: r} + got, err := p.IsEmpty(context.Background(), conn()) + if err != nil { + t.Fatalf("IsEmpty(%q): %v", tc.out, err) + } + if got != tc.want { + t.Errorf("IsEmpty(%q) = %v, want %v", tc.out, got, tc.want) + } + if r.cmds[0][0] != "psql" { + t.Errorf("IsEmpty shelled %q, want psql", r.cmds[0][0]) + } + } +} + +func TestPreflightMissingTool(t *testing.T) { + p := PgDumper{LookPath: func(string) (string, error) { return "", errors.New("not found") }} + err := p.Preflight(context.Background()) + if err == nil { + t.Fatal("Preflight should fail when the tool is absent") + } + if !IsToolMissing(err) { + t.Errorf("want ErrToolMissing, got %T: %v", err, err) + } + if !strings.Contains(err.Error(), "not found") || !strings.Contains(err.Error(), "postgresql-client") { + t.Errorf("remediation missing: %v", err) + } +} + +func TestPreflightPresent(t *testing.T) { + p := PgDumper{LookPath: func(string) (string, error) { return "/usr/bin/x", nil }} + if err := p.Preflight(context.Background()); err != nil { + t.Errorf("Preflight should pass when tools are present: %v", err) + } +} diff --git a/internal/orchestrate/snapshot.go b/internal/orchestrate/snapshot.go new file mode 100644 index 0000000..81f8159 --- /dev/null +++ b/internal/orchestrate/snapshot.go @@ -0,0 +1,313 @@ +package orchestrate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/open-source-cloud/devstack/internal/db" + "github.com/open-source-cloud/devstack/internal/generate" + "github.com/open-source-cloud/devstack/internal/lock" + "github.com/open-source-cloud/devstack/internal/store" +) + +// This file is the imperative side of spec 15 (thin v2 scope): Postgres-only +// `db snapshot` / `db restore` / `db snapshot ls` against a project's per-project +// tenant database on the SHARED Postgres. It reuses the provision phase's exact +// host-reachability pattern (engineTarget → FreeHostPort + writeProvisionOverlay +// + `compose up -d ` on the shared stack, DECISIONS D8) so the dump/restore +// client tooling reaches the warm server over a ledger-allocated 127.0.0.1 host +// port WITHOUT publishing a permanent one. +// +// Lock discipline (spec 15): the streaming dump/restore PROCESS runs OUTSIDE the +// flock (it is long — holding the lock for a multi-GB pg_restore would serialize +// every other invocation). Only the ledger row writes + the port allocation run +// inside the flock (FreeHostPort self-locks; the snapshot row write is wrapped). + +// snapshotKind is the free-text provisioned-ledger kind for a captured dump. The +// kind column is free-text so no migration is needed (spec 15 / task note). +const snapshotKind = "snapshot" + +// SnapshotOptions selects the tenant to capture. +type SnapshotOptions struct { + Project string // owner project (default: the workspace's single/first project) + Database string // physical tenant db (default: the project's own db) + Instance string // shared Postgres instance (default: the first postgres instance) + Name string // human label (default: a timestamp label) +} + +// RestoreOptions selects the tenant + snapshot to replay. +type RestoreOptions struct { + Project string + Database string + Instance string + Name string // required: the snapshot label to restore + Force bool // replay over a non-empty tenant (destructive) +} + +// SnapshotMeta is the on-disk + ledger record of one captured dump. It is written +// as a sidecar JSON next to the dump and surfaced verbatim by `db snapshot ls`. +type SnapshotMeta struct { + Name string `json:"name"` + Project string `json:"project"` + Kind string `json:"kind"` // pg (this milestone) + Instance string `json:"instance"` // shared instance captured from + Database string `json:"database"` // physical tenant db + Digest string `json:"digest"` // sha256 of the dump bytes + Size int64 `json:"size"` // dump bytes + CreatedAt string `json:"created_at"` + Path string `json:"path"` // absolute dump path +} + +// pgTenantDB maps a project name to its default tenant database identifier +// (hyphens → underscores), matching provision.EnsureProject's naming. +func pgTenantDB(project string) string { return strings.ReplaceAll(project, "-", "_") } + +// resolveTenant fills in the (project, database, instance) defaults and validates +// that a shared Postgres instance exists. +func resolveTenant(d UpDeps, project, database, instance string) (proj, dbName, inst string, err error) { + proj = project + if proj == "" { + if names := sortedProjects(d.Model); len(names) > 0 { + proj = names[0] + } + } + if proj == "" { + return "", "", "", fmt.Errorf("no project in this workspace to snapshot") + } + if _, ok := d.Model.Projects[proj]; !ok { + return "", "", "", fmt.Errorf("project %q is not in this workspace", proj) + } + inst = instance + if inst == "" { + var ok bool + inst, ok = ResolveInstance(d.Model, "postgres") + if !ok { + return "", "", "", fmt.Errorf("no shared postgres instance in this workspace (declare one under workspace.shared)") + } + } else if d.Model.Workspace.Shared[inst].Template != "postgres" { + return "", "", "", fmt.Errorf("shared instance %q is not a postgres engine", inst) + } + dbName = database + if dbName == "" { + dbName = pgTenantDB(proj) + } + return proj, dbName, inst, nil +} + +// tenantConn resolves the host-reachable admin endpoint for the tenant, reusing +// the provision overlay (allocates/looks up the ledger port, applies the loopback +// overlay via compose up). Returns the ConnInfo the dumper connects with. +func tenantConn(ctx context.Context, d UpDeps, inst, dbName string) (db.ConnInfo, error) { + target, err := engineTarget(ctx, d, "postgres", inst) + if err != nil { + return db.ConnInfo{}, err + } + return db.ConnInfo{ + Host: target.Host, + Port: target.Port, + User: target.AdminEnv["user"], + Password: target.AdminEnv["password"], + Database: dbName, + }, nil +} + +// Snapshot captures the project's tenant database to the workspace snapshot store +// and records a ledger row. The dump streams OUTSIDE the flock; only the ledger +// write is locked (spec 15). +func Snapshot(ctx context.Context, d UpDeps, dumper db.Dumper, opt SnapshotOptions) (SnapshotMeta, error) { + proj, dbName, inst, err := resolveTenant(d, opt.Project, opt.Database, opt.Instance) + if err != nil { + return SnapshotMeta{}, err + } + name := opt.Name + if name == "" { + name = time.Now().UTC().Format("20060102-150405") + } + if err := validSnapshotName(name); err != nil { + return SnapshotMeta{}, err + } + + conn, err := tenantConn(ctx, d, inst, dbName) + if err != nil { + return SnapshotMeta{}, err + } + + dir := store.SnapshotsPath(d.Model.Workspace.Name) + if err := os.MkdirAll(dir, 0o755); err != nil { + return SnapshotMeta{}, fmt.Errorf("create snapshot store: %w", err) + } + dumpPath := filepath.Join(dir, name+".dump") + + // The dump PROCESS runs outside the flock (spec 15 — long-running). + if err := dumper.Snapshot(ctx, conn, dumpPath); err != nil { + return SnapshotMeta{}, err + } + + digest, size, err := hashFile(dumpPath) + if err != nil { + return SnapshotMeta{}, err + } + meta := SnapshotMeta{ + Name: name, Project: proj, Kind: "pg", Instance: inst, Database: dbName, + Digest: digest, Size: size, CreatedAt: time.Now().UTC().Format(time.RFC3339), Path: dumpPath, + } + if err := writeSidecar(dir, meta); err != nil { + return SnapshotMeta{}, err + } + + // Ledger row + event, inside the flock (fast). + if err := lock.WithLock(ctx, d.LockPath, func() error { + if err := d.DB.RecordProvisioned(proj, snapshotKind, name); err != nil { + return err + } + d.DB.LogEvent("db.snapshot", proj, fmt.Sprintf("%s of %s (%s, %d bytes)", name, dbName, generate.SharedAlias(inst), size)) + return nil + }); err != nil { + return SnapshotMeta{}, err + } + return meta, nil +} + +// Restore replays a stored snapshot into the project's tenant database. It refuses +// a non-empty tenant unless opt.Force (data-loss guard, spec 15). The pg_restore +// PROCESS runs outside the flock; the event row write is locked. +func Restore(ctx context.Context, d UpDeps, dumper db.Dumper, opt RestoreOptions) (SnapshotMeta, error) { + if opt.Name == "" { + return SnapshotMeta{}, fmt.Errorf("a snapshot name is required") + } + proj, dbName, inst, err := resolveTenant(d, opt.Project, opt.Database, opt.Instance) + if err != nil { + return SnapshotMeta{}, err + } + dir := store.SnapshotsPath(d.Model.Workspace.Name) + meta, err := readSidecar(dir, opt.Name) + if err != nil { + return SnapshotMeta{}, err + } + if _, statErr := os.Stat(meta.Path); statErr != nil { + return SnapshotMeta{}, fmt.Errorf("snapshot dump %q is missing: %w", meta.Path, statErr) + } + // Integrity: re-hash the dump and hard-fail on mismatch (spec 15). + digest, _, err := hashFile(meta.Path) + if err != nil { + return SnapshotMeta{}, err + } + if meta.Digest != "" && digest != meta.Digest { + return SnapshotMeta{}, fmt.Errorf("snapshot %q is corrupted: digest %s does not match recorded %s", opt.Name, digest, meta.Digest) + } + + conn, err := tenantConn(ctx, d, inst, dbName) + if err != nil { + return SnapshotMeta{}, err + } + + if !opt.Force { + empty, err := dumper.IsEmpty(ctx, conn) + if err != nil { + return SnapshotMeta{}, err + } + if !empty { + return SnapshotMeta{}, fmt.Errorf("refusing to restore over non-empty database %q (data would be lost); pass --force to overwrite", dbName) + } + } + + // The restore PROCESS runs outside the flock (spec 15 — long-running). + if err := dumper.Restore(ctx, conn, meta.Path); err != nil { + return SnapshotMeta{}, err + } + + if err := lock.WithLock(ctx, d.LockPath, func() error { + d.DB.LogEvent("db.restore", proj, fmt.Sprintf("%s into %s (%s, digest %s)", opt.Name, dbName, generate.SharedAlias(inst), digest)) + return nil + }); err != nil { + return SnapshotMeta{}, err + } + return meta, nil +} + +// ListSnapshots returns the project's captured snapshots (lock-free): the ledger +// rows of kind=snapshot enriched with each dump's on-disk sidecar metadata. A +// missing sidecar still yields a row (name only) so a partially-removed store is +// visible rather than hidden. +func ListSnapshots(d UpDeps, project string) ([]SnapshotMeta, error) { + proj := project + if proj == "" { + if names := sortedProjects(d.Model); len(names) > 0 { + proj = names[0] + } + } + rows, err := d.DB.ProvisionedFor(proj) + if err != nil { + return nil, err + } + dir := store.SnapshotsPath(d.Model.Workspace.Name) + var out []SnapshotMeta + for _, r := range rows { + if r.Kind != snapshotKind { + continue + } + meta, err := readSidecar(dir, r.Name) + if err != nil { + // Sidecar gone — surface the bare ledger row. + meta = SnapshotMeta{Name: r.Name, Project: proj, Kind: "pg", CreatedAt: r.CreatedAt} + } + out = append(out, meta) + } + return out, nil +} + +// --- helpers --------------------------------------------------------------- + +func sidecarPath(dir, name string) string { return filepath.Join(dir, name+".json") } + +func writeSidecar(dir string, meta SnapshotMeta) error { + b, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + return os.WriteFile(sidecarPath(dir, meta.Name), b, 0o644) +} + +func readSidecar(dir, name string) (SnapshotMeta, error) { + b, err := os.ReadFile(sidecarPath(dir, name)) + if err != nil { + return SnapshotMeta{}, fmt.Errorf("no snapshot %q in the store: %w", name, err) + } + var meta SnapshotMeta + if err := json.Unmarshal(b, &meta); err != nil { + return SnapshotMeta{}, fmt.Errorf("read snapshot metadata %q: %w", name, err) + } + return meta, nil +} + +// hashFile returns the sha256 hex digest + byte size of a file. +func hashFile(path string) (string, int64, error) { + f, err := os.Open(path) + if err != nil { + return "", 0, fmt.Errorf("open dump %q: %w", path, err) + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + if err != nil { + return "", 0, fmt.Errorf("hash dump %q: %w", path, err) + } + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +// validSnapshotName rejects path-traversal and separators in the label so it maps +// cleanly to a single file in the store. +func validSnapshotName(name string) error { + if name == "" || strings.ContainsAny(name, `/\`) || strings.Contains(name, "..") { + return fmt.Errorf("invalid snapshot name %q (no slashes or '..')", name) + } + return nil +} diff --git a/internal/orchestrate/snapshot_test.go b/internal/orchestrate/snapshot_test.go new file mode 100644 index 0000000..f3f98f2 --- /dev/null +++ b/internal/orchestrate/snapshot_test.go @@ -0,0 +1,250 @@ +package orchestrate + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + dbpkg "github.com/open-source-cloud/devstack/internal/db" + "github.com/open-source-cloud/devstack/internal/store" +) + +// dumpRunner is a fake db.Runner: it records argv, materializes a dump file when +// it sees pg_dump --file (so digest/size work), replays it on pg_restore, and +// drives the psql emptiness probe from a configurable table count. +type dumpRunner struct { + cmds [][]string + envs [][]string + tables int // psql count(*) result for IsEmpty + restored bool // set when pg_restore ran +} + +func (r *dumpRunner) Run(_ context.Context, env []string, _, name string, args ...string) error { + r.cmds = append(r.cmds, append([]string{name}, args...)) + r.envs = append(r.envs, env) + if name == "pg_dump" { + // Honor --file : write a deterministic dump payload. + for i, a := range args { + if a == "--file" && i+1 < len(args) { + _ = os.WriteFile(args[i+1], []byte("PGDMP-fake-dump"), 0o644) + } + } + } + if name == "pg_restore" { + r.restored = true + } + return nil +} + +func (r *dumpRunner) Output(_ context.Context, env []string, _, name string, args ...string) ([]byte, error) { + r.cmds = append(r.cmds, append([]string{name}, args...)) + r.envs = append(r.envs, env) + if name == "psql" { + if r.tables == 0 { + return []byte("0\n"), nil + } + return []byte("5\n"), nil + } + return nil, nil +} + +func (r *dumpRunner) sawTool(tool string) []string { + for _, c := range r.cmds { + if c[0] == tool { + return c + } + } + return nil +} + +func newSnapEnv(t *testing.T) { + t.Helper() + // Isolate the snapshot store so we never touch the real ~/.devstack. + t.Setenv(store.HomeEnv, t.TempDir()) +} + +func TestSnapshotRestoreRoundTrip(t *testing.T) { + newSnapEnv(t) + d, fr, ledger := upFixture(t) + dr := &dumpRunner{} + dumper := dbpkg.PgDumper{Runner: dr, LookPath: func(string) (string, error) { return "/usr/bin/x", nil }} + + meta, err := Snapshot(context.Background(), d, dumper, SnapshotOptions{Project: "app", Name: "before"}) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + // pg_dump ran with the tenant db + custom format, over the loopback overlay. + pgd := dr.sawTool("pg_dump") + if pgd == nil { + t.Fatalf("pg_dump never ran: %v", dr.cmds) + } + joined := strings.Join(pgd, " ") + for _, want := range []string{"-h 127.0.0.1", "-d app", "--format=custom", "--no-owner", "--file"} { + if !strings.Contains(joined, want) { + t.Errorf("pg_dump argv missing %q: %s", want, joined) + } + } + // The password rode PGPASSWORD in the env, never on the argv. + if strings.Contains(joined, "PGPASSWORD") || strings.Contains(joined, "devstack") && strings.Contains(joined, "password") { + t.Errorf("password leaked into pg_dump argv: %s", joined) + } + if !slices.ContainsFunc(dr.envs, func(e []string) bool { + return slices.ContainsFunc(e, func(kv string) bool { return strings.HasPrefix(kv, "PGPASSWORD=") }) + }) { + t.Error("PGPASSWORD not passed via env") + } + + // The host-port overlay was allocated in the ledger and applied via compose up. + port, ok, _ := ledger.PortFor("shared-postgres", "pg-provision") + if !ok || port == 0 { + t.Errorf("host port not allocated for the snapshot overlay: port=%d ok=%v", port, ok) + } + if !fr.saw("-p devstack-shared", "compose.provision.yaml") { + t.Errorf("loopback overlay not applied via compose up: %v", fr.cmds) + } + + // The ledger recorded the snapshot row. + rows, _ := ledger.ProvisionedFor("app") + found := false + for _, r := range rows { + if r.Kind == snapshotKind && r.Name == "before" { + found = true + } + } + if !found { + t.Errorf("snapshot ledger row not recorded: %v", rows) + } + + // The dump + sidecar landed in the workspace store. + if _, err := os.Stat(meta.Path); err != nil { + t.Errorf("dump file missing: %v", err) + } + if meta.Digest == "" || meta.Size == 0 { + t.Errorf("meta digest/size not computed: %+v", meta) + } + wantDir := store.SnapshotsPath("demo") + if filepath.Dir(meta.Path) != wantDir { + t.Errorf("dump dir = %q, want %q", filepath.Dir(meta.Path), wantDir) + } + + // Restore round-trips (tenant is empty → no --force needed). + dr.tables = 0 + rmeta, err := Restore(context.Background(), d, dumper, RestoreOptions{Project: "app", Name: "before"}) + if err != nil { + t.Fatalf("Restore: %v", err) + } + if rmeta.Digest != meta.Digest { + t.Errorf("restore digest %q != snapshot digest %q", rmeta.Digest, meta.Digest) + } + pgr := dr.sawTool("pg_restore") + if pgr == nil { + t.Fatalf("pg_restore never ran: %v", dr.cmds) + } + rjoined := strings.Join(pgr, " ") + for _, want := range []string{"-d app", "--clean", "--if-exists", "--no-owner"} { + if !strings.Contains(rjoined, want) { + t.Errorf("pg_restore argv missing %q: %s", want, rjoined) + } + } + if !dr.restored { + t.Error("pg_restore was not invoked") + } +} + +func TestRestoreRefusesNonEmptyWithoutForce(t *testing.T) { + newSnapEnv(t) + d, _, _ := upFixture(t) + dr := &dumpRunner{} + dumper := dbpkg.PgDumper{Runner: dr, LookPath: func(string) (string, error) { return "/usr/bin/x", nil }} + + if _, err := Snapshot(context.Background(), d, dumper, SnapshotOptions{Project: "app", Name: "snap"}); err != nil { + t.Fatalf("Snapshot: %v", err) + } + + // Tenant now reports 5 tables → restore must refuse without --force. + dr.tables = 5 + _, err := Restore(context.Background(), d, dumper, RestoreOptions{Project: "app", Name: "snap"}) + if err == nil { + t.Fatal("Restore should refuse a non-empty tenant without --force") + } + if !strings.Contains(err.Error(), "non-empty") { + t.Errorf("unexpected error: %v", err) + } + // pg_restore must NOT have run. + if dr.restored { + t.Error("pg_restore ran despite the non-empty refusal") + } + + // With --force it proceeds. + if _, err := Restore(context.Background(), d, dumper, RestoreOptions{Project: "app", Name: "snap", Force: true}); err != nil { + t.Fatalf("Restore --force: %v", err) + } + if !dr.restored { + t.Error("pg_restore did not run under --force") + } +} + +// TestSnapshotJSONContract asserts the documented --json schema (name, kind, +// digest, size, created_at) round-trips through the marshaled SnapshotMeta — the +// non-TTY contract the `db snapshot --json` / `db snapshot ls --json` verbs emit. +func TestSnapshotJSONContract(t *testing.T) { + newSnapEnv(t) + d, _, _ := upFixture(t) + dr := &dumpRunner{} + dumper := dbpkg.PgDumper{Runner: dr, LookPath: func(string) (string, error) { return "/usr/bin/x", nil }} + + meta, err := Snapshot(context.Background(), d, dumper, SnapshotOptions{Project: "app", Name: "j1"}) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + b, err := json.Marshal(meta) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + for _, key := range []string{"name", "project", "kind", "database", "digest", "size", "created_at", "path"} { + if _, ok := got[key]; !ok { + t.Errorf("snapshot json missing key %q: %s", key, b) + } + } + if got["name"] != "j1" || got["kind"] != "pg" { + t.Errorf("unexpected json values: %s", b) + } +} + +func TestListSnapshots(t *testing.T) { + newSnapEnv(t) + d, _, _ := upFixture(t) + dr := &dumpRunner{} + dumper := dbpkg.PgDumper{Runner: dr, LookPath: func(string) (string, error) { return "/usr/bin/x", nil }} + + for _, n := range []string{"one", "two"} { + if _, err := Snapshot(context.Background(), d, dumper, SnapshotOptions{Project: "app", Name: n}); err != nil { + t.Fatalf("Snapshot %s: %v", n, err) + } + } + snaps, err := ListSnapshots(d, "app") + if err != nil { + t.Fatalf("ListSnapshots: %v", err) + } + if len(snaps) != 2 { + t.Fatalf("want 2 snapshots, got %d: %+v", len(snaps), snaps) + } + names := []string{snaps[0].Name, snaps[1].Name} + if !slices.Contains(names, "one") || !slices.Contains(names, "two") { + t.Errorf("snapshot names = %v, want one+two", names) + } + for _, s := range snaps { + if s.Database != "app" || s.Digest == "" { + t.Errorf("snapshot meta incomplete: %+v", s) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 5bb8449..bdd6374 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -31,6 +31,7 @@ const ( ConfigFile = "config.yaml" TemplatesDir = "templates" SharedDir = "shared" + SnapshotsDir = "snapshots" ) // Home returns the devstack home directory: $DEVSTACK_HOME if set, else @@ -56,6 +57,13 @@ func TemplatesPath() string { return filepath.Join(Home(), TemplatesDir) } // SharedPath is the global shared-stack artifacts directory. func SharedPath() string { return filepath.Join(Home(), SharedDir) } +// SnapshotsPath is the db-snapshot store for one workspace: dumps captured by +// `devstack db snapshot` live under ~/.devstack/snapshots// (spec 15). +// Keyed by workspace so two checkouts' snapshots never collide. +func SnapshotsPath(workspace string) string { + return filepath.Join(Home(), SnapshotsDir, workspace) +} + // Initialized reports whether the store config file exists. func Initialized() bool { info, err := os.Stat(ConfigPath())