From c5b7618f0a18881ddea68caf15d94440bb5aa7ed Mon Sep 17 00:00:00 2001 From: Steven Sklar Date: Thu, 27 Aug 2026 19:21:57 -0400 Subject: [PATCH] docs: add Operator restore and migration guide --- .../getting-started/restore-and-migrate.md | 552 ++++++++++++++++++ .../enterprise-kubernetes-operator/index.md | 2 + documentation/sidebars.js | 5 + 3 files changed, 559 insertions(+) create mode 100644 documentation/enterprise-kubernetes-operator/getting-started/restore-and-migrate.md diff --git a/documentation/enterprise-kubernetes-operator/getting-started/restore-and-migrate.md b/documentation/enterprise-kubernetes-operator/getting-started/restore-and-migrate.md new file mode 100644 index 000000000..1e53c2bcd --- /dev/null +++ b/documentation/enterprise-kubernetes-operator/getting-started/restore-and-migrate.md @@ -0,0 +1,552 @@ +--- +title: Restore or migrate QuestDB +description: + Restore QuestDB from an object-store backup or migrate an existing QuestDB + onto the Kubernetes Operator with a replica-first cutover. +--- + +# Restore or migrate QuestDB + +This guide covers two cloud-neutral ways to create a QuestDB Enterprise cluster +with existing data: + +1. **Restore:** create a new writable cluster from an object-store backup. +2. **Migrate:** create a replica-only follower of an external QuestDB, let it + catch up, and promote it after a controlled source drain. + +Both paths use the same Kubernetes resources on every supported cloud. The +provider-specific bucket or container, credentials, and pod identity are kept in +an existing `QuestDBObjectStore`. + +Before running a command, replace every `` value. An unreplaced +placeholder can be interpreted as shell redirection. + +## Before you start + +This guide assumes that: + +- the QuestDB Enterprise Kubernetes Operator is installed; +- the tenant namespace exists; +- the namespace has access to the QuestDB Enterprise image, either through an + `imagePullSecret` or ambient node credentials; +- a `ReadWriteOnce` StorageClass with `fsGroup` support is available; +- a same-namespace `QuestDBObjectStore` and any referenced credential Secret + already provide access to the source object store; and +- you know the source backup prefix and, for migration, the source replication + WAL prefix. + +See +[Configuration](/docs/enterprise-kubernetes-operator/configuration/#object-storage) +if object-store access is not ready. The operator does not test, list, read, or +write the store. QuestDB pods perform the object-store I/O, and the consuming +cluster's conditions are the readiness signal. + +Confirm the APIs, source store, and StorageClass before continuing: + +```sh +kubectl get crd questdbclusters.questdb.io \ + questdbobjectstores.questdb.io questdbpromotions.questdb.io +kubectl get questdbobjectstore -n +kubectl get storageclass +``` + +Copy the exact QuestDB Enterprise image and `imagePullSecrets` from a working +cluster when possible. Remove the `imagePullSecrets` block from the examples +only when every destination node has ambient pull access. + +## Scenario 1: Restore from an object-store backup + +A restore always creates a **new** `QuestDBCluster` and a new PVC. It never +restores over a running cluster or an existing volume. + +### 1. Collect the source details + +Record: + +- the source `QuestDBObjectStore` name; +- the backup prefix under that store; +- the source's backup instance name; and +- the desired namespace, cluster name, image, StorageClass, and volume size. + +For a running source, get its backup instance name directly from QuestDB: + +```sql +SELECT backup_instance_name(); +``` + +For an operator-managed source, it is also normally available in status: + +```sh +kubectl get questdbcluster -n \ + -o jsonpath='{.status.replication.seed.backupInstanceName}{"\n"}' +``` + +Copy the value exactly. Set `sourceInstanceName` whenever the backup prefix +contains more than one backup instance. If the value is omitted, the engine can +select the source only when the prefix contains exactly one instance. + +### 2. Choose destination prefixes + +The restored cluster must not write backups or replication WAL into another live +cluster's prefixes. In this example: + +- `` remains the read-only restore source; +- `backup///` is the restored cluster's new backup + prefix; and +- the omitted replication root defaults to the identity-scoped + `db///`. + +The example reuses the source `QuestDBObjectStore` for the restored cluster's +own backup and replication writes. To use a different object store, replace the +top-level `objectStoreRef` with another existing, writable store in the same +namespace. Keep `bootstrap.recovery.source.objectStoreRef` pointed at the source +store, and keep all destination prefixes distinct from live source prefixes. + +### 3. Create the restored cluster + +Save the following as `restore.yaml`: + +```yaml +apiVersion: questdb.io/v1alpha1 +kind: QuestDBCluster +metadata: + name: + namespace: +spec: + image: + imagePullSecrets: + - name: + storage: + storageClassName: + size: 100Gi + resources: + requests: + memory: 4Gi + limits: + memory: 4Gi + objectStoreRef: + name: + bootstrap: + recovery: + source: + objectStoreRef: + name: + root: + sourceInstanceName: + backup: + enabled: true + schedule: "0 * * * *" + timezone: UTC + retention: 5 + root: backup/// +``` + +`spec.bootstrap` is immutable. Review the store, prefix, and instance name +before applying the file: + +```sh +kubectl apply -f restore.yaml +``` + +The operator withholds the genesis pod until it can resolve the source store. +The QuestDB recovery init container then restores and validates the backup. The +operator never reads the backup itself. + +### 4. Watch the restore + +Use a bounded loop that stops on success or terminal recovery failure: + +```bash +RECOVERED="" +FAILED="" +for _ in $(seq 1 180); do + RECOVERED="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.conditions[?(@.type=="Recovered")].status}')" + FAILED="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.conditions[?(@.type=="RecoveryFailed")].status}')" + [ "$RECOVERED" = "True" ] && break + [ "$FAILED" = "True" ] && break + sleep 10 +done +kubectl get questdbcluster -n \ + -o jsonpath='{range .status.conditions[*]}{.type}{"="}{.status}{"/"}{.reason}{" "}{.message}{"\n"}{end}' +[ "$RECOVERED" = "True" ] && [ "$FAILED" != "True" ] +``` + +If the final command fails, do not patch `spec.bootstrap` or reuse the PVC. +Follow +[restore failure cleanup](/docs/enterprise-kubernetes-operator/operations/backup-restore/#if-it-fails) +and create a fresh cluster with corrected immutable values. + +### 5. Verify the restored writer and data + +`Recovered=True` proves that the engine completed recovery. Also require the +current writer-health contract: + +```bash +GENERATION="$(kubectl get questdbcluster -n \ + -o jsonpath='{.metadata.generation}')" +for _ in $(seq 1 120); do + STATE="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.observedGeneration}{"|"}{range .status.conditions[?(@.type=="Available")]}{.status}{"/"}{.reason}{end}{"|"}{range .status.conditions[?(@.type=="Progressing")]}{.status}{"/"}{.reason}{end}{"|"}{range .status.conditions[?(@.type=="WriteHealthy")]}{.status}{"/"}{.reason}{end}')" + IFS='|' read -r OBSERVED AVAILABLE PROGRESSING WRITE_HEALTHY <<< "$STATE" + if [ "$OBSERVED" = "$GENERATION" ] && \ + [ "$AVAILABLE" = "True/PrimaryReady" ] && \ + [ "$PROGRESSING" = "False/Settled" ] && \ + [ "$WRITE_HEALTHY" = "True/Healthy" ]; then + break + fi + sleep 10 +done +printf 'observed=%s available=%s progressing=%s writeHealthy=%s\n' \ + "$OBSERVED" "$AVAILABLE" "$PROGRESSING" "$WRITE_HEALTHY" +[ "$OBSERVED" = "$GENERATION" ] && \ +[ "$AVAILABLE" = "True/PrimaryReady" ] && \ +[ "$PROGRESSING" = "False/Settled" ] && \ +[ "$WRITE_HEALTHY" = "True/Healthy" ] +``` + +Confirm that the RW Service has an endpoint: + +```sh +kubectl get endpointslice -n \ + -l kubernetes.io/service-name=-rw +``` + +Before sending application traffic, connect through `-rw` and +validate critical tables, expected row counts, minimum and maximum timestamps, +application invariants, and free storage. See +[Connect to a database](/docs/enterprise-kubernetes-operator/operations/database/#connect) +for a temporary PGWire connection. + +For a point-in-time restore, add `bootstrap.recovery.recoveryTarget` when the +cluster is first created. See +[Point-in-time recovery](/docs/enterprise-kubernetes-operator/operations/backup-restore/#point-in-time-recovery-pitr) +for its retained-window and timestamp rules. + +## Scenario 2: Migrate an external QuestDB with a follower + +This path keeps the external source writable while an operator-managed replica +restores its backup and consumes its replication WAL. Cutover downtime is +limited to stopping and draining the source, consuming the final WAL, and +promoting the follower. + +The operator does not connect to, configure, stop, or fence the external source. +Those steps remain your responsibility. + +### 1. Prepare the source + +Before creating the follower, confirm that the external source: + +- runs a QuestDB Enterprise version compatible with the destination image; +- has a completed backup under a known backup prefix; +- uploads replication WAL under a known WAL prefix in the same object store; +- retains WAL back to the seed backup; +- returns an exact, non-empty value from `SELECT backup_instance_name();`; and +- can be stopped and restarted once with + `replication.role=primary-catchup-uploads` during cutover. + +The backup instance name must match `^[a-z0-9]+(-[a-z0-9]+)*$`. Confirm the +backup prefix, WAL prefix, and instance name against the source configuration; +they become immutable on the follower. + +### 2. Create a replica-only follower + +Save the following as `follower.yaml`: + +```yaml +apiVersion: questdb.io/v1alpha1 +kind: QuestDBCluster +metadata: + name: + namespace: +spec: + image: + imagePullSecrets: + - name: + instances: 1 + storage: + storageClassName: + size: 100Gi + resources: + requests: + memory: 4Gi + limits: + memory: 4Gi + objectStoreRef: + name: + backup: + enabled: true + schedule: "0 * * * *" + timezone: UTC + retention: 5 + root: + replication: + root: + bootstrap: + follow: + sourceInstanceName: +``` + +While the cluster is following, every instance is a replica and the backup +scheduler is paused. After promotion, this cluster adopts the source prefixes +and begins taking its own backups there. + +Review all immutable source selectors, then apply the file: + +```sh +kubectl apply -f follower.yaml +``` + +### 3. Wait for the follower to serve reads + +Wait until the replica has restored its baseline and reconciliation is settled: + +```bash +for _ in $(seq 1 180); do + STATE="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.replication.following}{"|"}{.status.readyInstances}{"|"}{range .status.conditions[?(@.type=="Available")]}{.status}{"/"}{.reason}{end}{"|"}{range .status.conditions[?(@.type=="Progressing")]}{.status}{"/"}{.reason}{end}')" + IFS='|' read -r FOLLOWING READY AVAILABLE PROGRESSING <<< "$STATE" + if [ "$FOLLOWING" = "true" ] && [ "$READY" = "1" ] && \ + [ "$AVAILABLE" = "True/Following" ] && \ + [ "$PROGRESSING" = "False/Settled" ]; then + break + fi + sleep 10 +done +printf 'following=%s ready=%s available=%s progressing=%s\n' \ + "$FOLLOWING" "$READY" "$AVAILABLE" "$PROGRESSING" +[ "$FOLLOWING" = "true" ] && [ "$READY" = "1" ] && \ +[ "$AVAILABLE" = "True/Following" ] && \ +[ "$PROGRESSING" = "False/Settled" ] +``` + +A healthy follower deliberately has no current primary and no RW endpoint. +Confirm both properties: + +```sh +kubectl get questdbcluster -n \ + -o jsonpath='following={.status.replication.following}{" primary="}{.status.currentPrimary}{"\n"}' +kubectl get endpointslice -n \ + -l kubernetes.io/service-name=-rw \ + -o jsonpath='{range .items[*].endpoints[*]}{.addresses}{"\n"}{end}' +``` + +The second command must print no endpoint addresses. + +### 4. Confirm replication catch-up + +Inspect the live follower position: + +```sh +kubectl get questdbcluster -n \ + -o jsonpath='{range .status.replication.replicas[*]}{.instance}{" caughtUpNow="}{.caughtUpNow}{" lagTxns="}{.lagTxns}{" suspended="}{.suspendedTables}{"\n"}{end}{range .status.conditions[?(@.type=="ReplicationHealthy")]}ReplicationHealthy={.status}{"/"}{.reason}{" "}{.message}{"\n"}{end}{.status.replication.stream}{"\n"}' +``` + +Prefer to begin cutover with `caughtUpNow=true` and `lagTxns=0`. A busy source +may briefly move away from zero. A quiet source may report `StreamNotDetermined` +because the engine omits already-caught-up tables from its poll; that is not +proof of success. In that case, reconfirm the immutable source identity and +roots, then query `-ro` and verify a recent, known source +record. + +Do not proceed with `ReplicationHealthy=False`, suspended tables, a known +backlog that is not advancing, or unverified source selectors. The planned +promotion performs a final fail-closed check after the source is drained. + +### 5. Stop writes and drain the source + +Record the start of cutover in the Bash shell you will keep open: + +```bash +CUTOVER_TIME_CAPTURED=false +CUTOVER_STARTED_AT="" +if CUTOVER_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" && \ + [ -n "$CUTOVER_STARTED_AT" ]; then + CUTOVER_TIME_CAPTURED=true +fi +[ "$CUTOVER_TIME_CAPTURED" = true ] && \ + printf 'Cutover started at %s\n' "$CUTOVER_STARTED_AT" +``` + +Continue in this shell only if `CUTOVER_TIME_CAPTURED=true`; the final backup +check rejects a missing timestamp. + +Then perform these steps with the external source's service manager or container +runtime: + +1. Stop all application writes to the external source. +2. Stop the source QuestDB process. +3. Configure the source to start once with + `replication.role=primary-catchup-uploads`. +4. Start the source and watch its logs. +5. Wait for `CLOSE_REASON_UPLOADS_COMPLETE_SUCCESS`. A normal shutdown without + this close reason does not prove that the final WAL reached object storage. +6. Confirm the source process has exited, and disable automatic restarts. + +The final upload has no safe fixed timeout. Supervise it at the source until it +succeeds. + +:::danger Do not promote while the external source may still be running as a +primary. The operator cannot fence an unmanaged process. Keep the old data +available for rollback investigation, but ensure the process and its supervisor +cannot restart it. ::: + +### 6. Promote the follower + +Create a one-shot planned promotion targeting the follower's instance serial +`1`: + +```sh +kubectl apply -f - < + namespace: +spec: + clusterRef: + name: + target: 1 + mode: Planned + catchUpTimeoutSeconds: 900 + primaryGracePeriodSeconds: 120 +EOF +``` + +The promotion waits for the source stream to remain quiet for at least 60 +seconds and for the target to consume the published WAL. It fails closed rather +than silently accepting a lossy cutover. + +Watch until the promotion completes or fails: + +```bash +PHASE="" +for _ in $(seq 1 180); do + PHASE="$(kubectl get questdbpromotion -n \ + -o jsonpath='{.status.phase}')" + printf '%s %s\n' "$(date -u +%FT%TZ)" "$PHASE" + case "$PHASE" in + Completed|Failed) break ;; + esac + sleep 10 +done +kubectl get questdbpromotion -n \ + -o jsonpath='{.status.phase}{" "}{.status.reason}{": "}{.status.message}{"\n"}{range .status.conditions[*]}{.type}{"="}{.status}{"/"}{.reason}{" "}{.message}{"\n"}{end}' +[ "$PHASE" = "Completed" ] +``` + +If it fails, leave the source stopped and read the reported reason before taking +another action. A failed promotion is terminal; correct the cause and create a +new promotion object. Do not remove the promotion finalizer. See +[If promotion stalls or fails](/docs/enterprise-kubernetes-operator/high-availability/#if-promotion-stalls-or-fails). + +### 7. Verify the new primary + +Wait for the promoted cluster's writer-health contract: + +```bash +GENERATION="$(kubectl get questdbcluster -n \ + -o jsonpath='{.metadata.generation}')" +for _ in $(seq 1 120); do + STATE="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.observedGeneration}{"|"}{.status.currentPrimary}{"|"}{.status.replication.following}{"|"}{range .status.conditions[?(@.type=="Available")]}{.status}{"/"}{.reason}{end}{"|"}{range .status.conditions[?(@.type=="Progressing")]}{.status}{"/"}{.reason}{end}{"|"}{range .status.conditions[?(@.type=="WriteHealthy")]}{.status}{"/"}{.reason}{end}')" + IFS='|' read -r OBSERVED PRIMARY FOLLOWING AVAILABLE PROGRESSING WRITE_HEALTHY <<< "$STATE" + if [ "$OBSERVED" = "$GENERATION" ] && \ + [ "$PRIMARY" = "-1" ] && \ + [ "$FOLLOWING" != "true" ] && \ + [ "$AVAILABLE" = "True/PrimaryReady" ] && \ + [ "$PROGRESSING" = "False/Settled" ] && \ + [ "$WRITE_HEALTHY" = "True/Healthy" ]; then + break + fi + sleep 10 +done +printf 'observed=%s primary=%s following=%s available=%s progressing=%s writeHealthy=%s\n' \ + "$OBSERVED" "$PRIMARY" "$FOLLOWING" "$AVAILABLE" "$PROGRESSING" "$WRITE_HEALTHY" +[ "$OBSERVED" = "$GENERATION" ] && \ +[ "$PRIMARY" = "-1" ] && \ +[ "$FOLLOWING" != "true" ] && \ +[ "$AVAILABLE" = "True/PrimaryReady" ] && \ +[ "$PROGRESSING" = "False/Settled" ] && \ +[ "$WRITE_HEALTHY" = "True/Healthy" ] +``` + +Confirm that `-rw` now has an endpoint, then connect through +that Service and validate recent data and application writes: + +```sh +kubectl get endpointslice -n \ + -l kubernetes.io/service-name=-rw +``` + +Permanently decommission the old source so that it cannot restart and contend +for the adopted WAL stream. + +### 8. Verify the first post-cutover backup + +The WAL cleaner remains held until the promoted cluster completes its own first +backup. With the hourly schedule in this guide, allow one schedule interval plus +the operator's roughly two-minute observation delay: + +```bash +PRIMARY_UID_CAPTURED=false +PRIMARY_UID_BEFORE_BACKUP="" +if PRIMARY_UID_BEFORE_BACKUP="$(kubectl get pod -1 \ + -n -o jsonpath='{.metadata.uid}')" && \ + [ -n "$PRIMARY_UID_BEFORE_BACKUP" ]; then + PRIMARY_UID_CAPTURED=true +fi + +BACKUP_VERIFIED=false +STATUS="" +END_TIME="" +for _ in $(seq 1 450); do + STATUS="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.backup.lastBackup.status}')" + END_TIME="$(kubectl get questdbcluster -n \ + -o jsonpath='{.status.backup.lastBackup.endTime}')" + if [ "${CUTOVER_TIME_CAPTURED:-false}" = true ] && \ + [ -n "$CUTOVER_STARTED_AT" ] && \ + [ "$STATUS" = "completed" ] && [ -n "$END_TIME" ] && \ + [[ "$END_TIME" > "$CUTOVER_STARTED_AT" ]]; then + BACKUP_VERIFIED=true + break + fi + [ "$STATUS" = "failed" ] && break + sleep 10 +done +printf 'status=%s endTime=%s cutoverStartedAt=%s\n' \ + "$STATUS" "$END_TIME" "$CUTOVER_STARTED_AT" +[ "${CUTOVER_TIME_CAPTURED:-false}" = true ] && \ +[ -n "$CUTOVER_STARTED_AT" ] && [ "$BACKUP_VERIFIED" = true ] +``` + +Releasing the WAL cleaner rolls the primary once. Prove that the asynchronous +roll occurred by waiting for the pod UID to change: + +```bash +PRIMARY_ROLLED=false +PRIMARY_UID_AFTER_BACKUP="" +for _ in $(seq 1 120); do + if PRIMARY_UID_AFTER_BACKUP="$(kubectl get pod -1 \ + -n -o jsonpath='{.metadata.uid}' 2>/dev/null)" && \ + [ "$PRIMARY_UID_CAPTURED" = true ] && \ + [ -n "$PRIMARY_UID_AFTER_BACKUP" ] && \ + [ "$PRIMARY_UID_AFTER_BACKUP" != "$PRIMARY_UID_BEFORE_BACKUP" ]; then + PRIMARY_ROLLED=true + break + fi + sleep 10 +done +printf 'before=%s after=%s rolled=%s\n' \ + "$PRIMARY_UID_BEFORE_BACKUP" "$PRIMARY_UID_AFTER_BACKUP" "$PRIMARY_ROLLED" +[ "$PRIMARY_UID_CAPTURED" = true ] && [ "$PRIMARY_ROLLED" = true ] +``` + +After the new pod appears, repeat the writer-health check from the previous step +and require it to settle before declaring the migration complete. + +For emergency source loss, multiple followers, or detailed failure handling, use +the full +[high-availability migration runbook](/docs/enterprise-kubernetes-operator/high-availability/#migrate-an-existing-questdb-onto-the-operator). diff --git a/documentation/enterprise-kubernetes-operator/index.md b/documentation/enterprise-kubernetes-operator/index.md index 2951361d8..5f46f6d15 100644 --- a/documentation/enterprise-kubernetes-operator/index.md +++ b/documentation/enterprise-kubernetes-operator/index.md @@ -37,6 +37,8 @@ static Secret. QuestDB database pods perform all object-store I/O. [AKS onboarding guide](/docs/enterprise-kubernetes-operator/getting-started/azure/). - **Shared install requirements:** see [Installation](/docs/enterprise-kubernetes-operator/installation/). +- **Restore or migrate existing data:** follow the cloud-neutral + [restore and migration guide](/docs/enterprise-kubernetes-operator/getting-started/restore-and-migrate/). - **PGWire TLS and network isolation:** plan them before creation with [Configuration](/docs/enterprise-kubernetes-operator/configuration/#pgwire-tls). - **Operate the operator:** use the diff --git a/documentation/sidebars.js b/documentation/sidebars.js index 84ffb0548..e82bca8cf 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -831,6 +831,11 @@ module.exports = { id: "enterprise-kubernetes-operator/getting-started/azure", label: "Azure AKS", }, + { + type: "doc", + id: "enterprise-kubernetes-operator/getting-started/restore-and-migrate", + label: "Restore or migrate", + }, ], }, "enterprise-kubernetes-operator/configuration",