diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 063a4c5..7186195 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,31 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev - run: cargo check --workspace --all-targets + # Every Dockerfile must build with the declared MSRV. + # + # A-36 raised `pangolin/Dockerfile` to match and missed + # `Dockerfile.tools`, which stayed on rust:1.88 and failed to build the + # instant the workspace MSRV moved past it - discovered while publishing + # 0.8.0, after the API image had already been pushed. + - name: Dockerfiles must pin the declared MSRV + working-directory: . + run: | + set -euo pipefail + declared=$(grep -m1 '^rust-version' pangolin/Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + echo "declared MSRV: $declared" + status=0 + while IFS= read -r dockerfile; do + pinned=$(grep -oE '^FROM rust:[0-9]+\.[0-9]+' "$dockerfile" | head -1 | sed 's/FROM rust://') || true + [ -z "$pinned" ] && continue + if [ "$pinned" != "$declared" ]; then + echo "::error file=$dockerfile::pins rust:$pinned but the workspace declares $declared" + status=1 + else + echo " ok $dockerfile -> rust:$pinned" + fi + done < <(find . -name 'Dockerfile*' -not -path './node_modules/*' -not -path '*/node_modules/*' -not -path './pangolin/target/*') + exit $status + helm: name: helm lint runs-on: ubuntu-latest @@ -612,6 +637,15 @@ jobs: working-directory: pangolin run: ./scripts/check_env_var_docs.sh + # The documentation carried 274 broken relative links: files moved between + # `docs/` subdirectories and the links pointing at them were never + # updated, plus references into a `planning/` directory that is not in + # this repository. A link that 404s is worse than no link - it sends a + # reader after something that does not exist while making the surrounding + # text look maintained. + - name: Every documentation link must resolve + run: ./scripts/check_doc_links.sh + - name: Every artifact must carry the same version run: | # Improvement #8. The "one version everywhere" property introduced in diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bbcf49..aa18c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,38 @@ Production-readiness work: authentication hardening, credential encryption, transactional correctness, MongoDB index management, the missing Iceberg operations, operational tooling, and OpenID Connect. +### Fixed — the published container images + +Four defects found by publishing 0.8.0 and then *running* what was published. +All 18 CI jobs passed over every one of them, because the `docker` job builds +images and does not exercise what it built. + +- **`Dockerfile.tools` still pinned `rust:1.88`.** A-36 raised the API image to + match the workspace MSRV and missed the CLI image, which then failed to + compile the moment `rust-version` moved to 1.94 — mid-release, after the API + image had already been pushed. The `msrv` job now fails if any Dockerfile + pins a version other than the declared one. +- **The CLI runtime stage installed `libssl-dev`,** the development package, + shipping OpenSSL headers and static archives in the published artefact. Now + `libssl3`. This is the same defect A-36 corrected in the API image. +- **The UI runtime stage copied the entire `node_modules`,** publishing all 22 + devDependencies — vite, playwright, vitest, svelte-check, the tailwind + toolchain — in the shipped image. Pruned to the single production dependency: + 283MB to 141MB locally, and `node_modules` from the full toolchain to 2.3MB. +- **Neither CLI accepted `--version`.** `pangolin-admin --version` was a clap + parse error, so there was no way to ask a binary which build it was — on a + tool distributed mainly as a container image, where `latest` tells you + nothing. + +Both images also ran as root and carried no OCI labels; both now run +unprivileged. + +The release script's overwrite guard was all-or-nothing: if any of the three +tags existed it refused to start, so the partial 0.8.0 failure could only be +finished with `ALLOW_OVERWRITE=1` — which would also have re-pushed the good +API image over itself. It now skips images already published at the target +version. + ### Added — operations: replicas, backup, performance **The token-cleanup job never ran.** `start_token_cleanup_job` was defined, the diff --git a/ORGANIZATION.md b/ORGANIZATION.md index c8d7e63..c057502 100644 --- a/ORGANIZATION.md +++ b/ORGANIZATION.md @@ -8,11 +8,19 @@ The Pangolin repository follows a monorepo structure separating the core Rust im pangolin-monorepo/ ├── docs/ # Comprehensive Documentation │ ├── api/ # API Reference and Swagger info +│ ├── architecture/ # Design and internals +│ ├── backend_storage/ # Per-backend storage notes +│ ├── best-practices/ # Deployment and usage guidance │ ├── cli/ # CLI Command Reference │ ├── features/ # Feature Guides (RBAC, Federation, etc.) │ ├── getting-started/ # Installation and Architecture guides │ ├── known-issues/ # Registry of quirks and temporary traps -│ └── ui/ # UI User Guide +│ ├── operations/ # Running it: parity, encryption, backup, +│ │ # performance, replicas, OIDC, runbook +│ ├── reference/ # Lookup tables and reference material +│ ├── ui/ # UI User Guide +│ ├── upgrading/ # Version-to-version upgrade notes +│ └── warehouse/ # Warehouse and credential-vending docs │ ├── pangolin/ # Core Rust Implementation (Workspace) │ ├── pangolin_api/ # REST API Server (Axum) @@ -27,13 +35,17 @@ pangolin-monorepo/ │ └── src/lib/ # Shared Components and Stores │ ├── pypangolin/ # Python SDK -│ ├── pypangolin/ # Source Code +│ ├── src/pypangolin/ # Source Code │ └── docs/ # SDK-specific Documentation │ ├── scripts/ # Automation & Verification │ ├── verify_pypangolin_*.py # SDK Verification Scripts -│ ├── test_release_*.py # End-to-End Release Tests -│ └── docker-build.sh # Build helpers +│ ├── release_smoke_test.py # Verifies a released image over HTTP +│ ├── backup_restore_drill.sh # Dumps, destroys, restores, verifies +│ ├── load_test.py # Load harness (client + server-side latency) +│ ├── bump_version.sh # One version across every artifact +│ ├── check_env_var_docs.sh # Env-var reference vs. the code +│ └── build_docker_sequential.sh # Builds and pushes the three images │ ├── tests/ # Integration Test Suites │ └── pyiceberg/ # PyIceberg compatibility tests diff --git a/README.md b/README.md index 066e9c1..17ca4d0 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ See [Quick Start Guide](docs/getting-started/getting_started.md) for detailed se *Production guides and operational wisdom.* - **[Production Runbook](docs/operations/runbook.md)** - Health, metrics, incidents, upgrades, backup. - **[Backend Feature Parity](docs/operations/backend-parity.md)** - Which features work on which backend. -- **[OAuth / SSO](docs/operations/oidc.md)** - Configuration, the 0.6.0 client change, and OIDC limitations. +- **[OAuth / OIDC](docs/operations/oidc.md)** - Provider setup, what is verified, and what still is not. - **[Best Practices Index](docs/best-practices/README.md)** - Complete guide to operating Pangolin. - **[Deployment & Security](docs/best-practices/deployment.md)** - Production checklists. - **[Scalability](docs/best-practices/scalability.md)** - Tuning for high performance. @@ -126,37 +126,44 @@ See [Quick Start Guide](docs/getting-started/getting_started.md) for detailed se ## 🚦 Project Status -**Current version: 0.6.0. Status: Alpha.** +**Current version: 0.8.0. Status: Beta.** -Pangolin is pre-1.0 software under active hardening. It is a capable catalog -with a broad feature set, and it is not yet something we would tell you to put -in front of a production data lake without reading the rest of this section. +Pangolin is pre-1.0 software. It is a capable catalog with a broad feature set, +and after two full audits it is substantially hardened — but see the honest +limits below and in [STATUS.md](STATUS.md) before putting it in front of a +production data lake. -0.6.0 is a **security release**. If you run anything earlier, upgrade: it fixes +**0.8.0 and 0.7.0 are security releases. If you run anything earlier, upgrade.** +Between them they fix a privilege escalation exploitable by any authenticated +principal, unauthorized cloud-credential vending, a logout that revoked nothing, a remotely exploitable OAuth account-takeover path, a working default JWT -signing secret published in this repository, an authentication bypass, an -unauthenticated denial-of-service primitive, and an Iceberg commit path that -could silently fork snapshot lineage under concurrent writers. See +signing secret published in this repository, and an authentication bypass. See [SECURITY.md](SECURITY.md) for the full list and the upgrade steps. +Note that **no 0.6.0 or 0.7.0 container image was ever published** — the release +pipeline could not complete. If you are running a Pangolin image older than +0.8.0, you are on 0.5.1 or earlier and predate every fix above. + ### Maturity by area | Area | Maturity | Notes | |---|---|---| | Iceberg REST — namespaces, tables, commits | **Solid** | Commit requirements including `assert-ref-snapshot-id` are enforced; unsupported operations return an error rather than a false `200 OK` | -| Iceberg REST — full spec coverage | **Partial** | Several endpoints are missing; see below | +| Iceberg REST — full spec coverage | **Good** | `registerTable`, `listViews`, `viewExists`, `dropView` added in 0.8.0. `commitTransaction` is deliberately absent and `replaceView`/`renameView` are not implemented; see below | | Multi-tenancy and isolation | **Solid** | Tenant scope is a required parameter throughout; isolation tests pass against the production middleware | | Git-style branching, tags, merge | **Good** | Merge direction and branch-asset tracking were fixed in 0.6.0 | | RBAC, service users, API keys | **Good** | API keys carry a key ID, so authentication is one bcrypt verification rather than a scan | +| Authentication | **Good** | OIDC with PKCE, `id_token` validation via JWKS, and `iss`/`aud`/`exp`/`nonce` checks from 0.8.0. Rate limited per address and per account. GitHub is not an OIDC provider and cannot be validated this way | | Audit logging | **Good** | 40+ actions, 19 resource types, plus authentication events from 0.6.0. Writes are best-effort and are not tamper-evident | | Observability | **New in 0.6.0** | Prometheus metrics, request IDs, working `RUST_LOG`, real health endpoints | | PostgreSQL backend | **Good** | The recommended backend. Provisioning from a fresh database was broken before 0.6.0 | | SQLite backend | **Good** | Single-writer; suitable for one node | -| MongoDB backend | **Beta** | No index management, no transactions, four known-failing tests | +| MongoDB backend | **Beta** | Index management and uniqueness constraints from 0.8.0. Still no versioned schema migrations, and multi-statement transactions only where the deployment provides a session | | Kubernetes deployment | **Good** | The chart shipped referencing three templates that did not exist; all present and CI-linted from 0.6.0 | -| Transactions for admin operations | **Partial** | PostgreSQL wraps `delete_catalog`, `delete_branch` and `merge_branch`; MongoDB wraps `delete_catalog` where the deployment supports sessions. Branch creation by copy is still not atomic | +| Transactions for admin operations | **Good** | PostgreSQL and SQLite wrap `delete_catalog`, `delete_branch`, `merge_branch` and branch-creation-by-copy; MongoDB wraps `delete_catalog` where the deployment supports sessions, and reports the non-atomic fallback rather than hiding it | | HA at N > 1 replicas | **Partial** | See below | -| Backup / restore / DR | **Undocumented and untested** | | +| Backup / restore / DR | **Documented and drilled** | `scripts/backup_restore_drill.sh` dumps, destroys and restores against a real database. Measured figures in [docs/operations/backup-and-recovery.md](docs/operations/backup-and-recovery.md). No point-in-time recovery | +| Warehouse credentials at rest | **Good** | AES-256-GCM when `PANGOLIN_ENCRYPTION_KEY` is set; plaintext with a startup warning when it is not | ### Known limitations diff --git a/SECURITY.md b/SECURITY.md index ad3976c..be9fcbd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -135,6 +135,12 @@ configured", which at least failed closed. Fixed in 0.7.0, with a CI job that builds every optional feature so it cannot recur silently. +### Upgrading + +- **To 0.8.0:** [docs/upgrading/0.7-to-0.8.md](docs/upgrading/0.7-to-0.8.md) — + MSRV 1.94, `PANGOLIN_ENCRYPTION_KEY`, MongoDB uniqueness constraints that can + fail on existing duplicates, rate limiting behind a proxy, and OIDC. + ### Upgrading to 0.7.0 1. **Rotate every issued token.** B0a means any account may have minted a diff --git a/STATUS.md b/STATUS.md index e0aa868..6d1d87b 100644 --- a/STATUS.md +++ b/STATUS.md @@ -12,10 +12,12 @@ pointing here. **Where they disagree with this file, this file is correct.** Everything marked done below is verified by tests that run against live PostgreSQL, MongoDB and MinIO, not by inspection: -- **63 test targets, 415 tests, zero failures** -- **19 CI jobs green**, including an authorization matrix, a four-backend parity - suite, both MongoDB topologies, an MSRV check, and a build of every optional - feature +- **65 test targets, 448 tests, zero failures** +- **18 CI jobs green on every push and pull request** — 14 job definitions, of + which `test` and `features` are matrices that expand to 2 and 4 runs. They + include an authorization matrix, a four-backend parity suite, both MongoDB + topologies, an MSRV check, and a build of every optional feature. Five further + jobs build and release the binaries, and run only on a `v*` tag. - `cargo audit` clean; clippy at a ratcheted budget of 30 (from 314) That standard exists because this project has repeatedly had things that @@ -51,7 +53,7 @@ exited 25 seconds after startup passed all 18 CI jobs and the full suite. | Item | Where | |---|---| -| CI that actually runs — 19 jobs | 0.7.0 / 0.8.0 | +| CI that actually runs — 18 jobs per push, 5 more per release tag | 0.7.0 / 0.8.0 | | A release pipeline that produces a release (it never had; `macos-13` was retired and hung every tag for 24h) | 0.7.0 | | A release gate that verifies the published image over HTTP | 0.7.0 | | Token-cleanup sweep that **runs** (it was dead code) and staggers across replicas | 0.8.0 | @@ -119,18 +121,48 @@ blip, every revoked token is accepted again. Watch - Eight accepted dependency advisories to re-check when dependencies move - clippy 30 and svelte-check 150 backlogs, both ratcheted -## Not shipped +## Shipped -**0.7.0 and 0.8.0 are not published.** The work is merged to the branch and CI -is green, but the merge, tag, Docker push and PyPI upload have not been made. -The most recent published artifact is `alexmerced/pangolin-api:0.5.1` from -2025-12-30 — so **anything running Pangolin today is on 0.5.1**, which predates -every security fix listed above. +**0.8.0 is published**, on 2026-08-11: -The `SECURITY.md` advisory covers `< 0.7.0` for that reason. +| Artifact | State | +|---|---| +| GitHub release `v0.8.0` | 12 binaries across linux, macOS Intel, macOS ARM and Windows — **the first release this project has produced**; v0.4.0 through v0.6.0 had tags and no releases | +| PyPI `pypangolin` 0.8.0 | wheel and sdist | +| `alexmerced/pangolin-api` 0.8.0 + latest | linux/amd64 + linux/arm64, 58MB | +| `alexmerced/pangolin-cli` 0.8.0 + latest | linux/amd64 + linux/arm64, 39MB — the first CLI image since 0.5.0 | +| `alexmerced/pangolin-ui` 0.8.0 + latest | linux/amd64 + linux/arm64, 52MB — the first UI image since 0.5.0 | + +Each was verified by pulling the published tag and running it, not by trusting +the build's exit code: the CLI reports `pangolin-admin 0.8.0` and runs as uid +10001, the UI serves `HTTP 200` as uid 1000 with a 2.3MB `node_modules`. -The PyPI token has been rotated. Still requiring a person: decide whether to -publish a GHSA once a fixed version actually exists. +Publishing them turned up four defects that every one of the 18 CI jobs had +passed over, because CI builds images and never runs what it built: + +| Defect | Consequence | +|---|---| +| `Dockerfile.tools` still pinned `rust:1.88` | The CLI image could not compile once the MSRV moved to 1.94. It failed mid-release, after the API image had already pushed. | +| The CLI runtime stage installed `libssl-dev` | Headers and static archives shipped in the published artefact. A-36 fixed this in the API image and missed this one. | +| The UI runtime stage copied all of `node_modules` | All 22 devDependencies — vite, playwright, vitest, svelte-check, tailwind — published in the image. It is now 2.3MB. | +| Neither CLI accepted `--version` | No way to ask a binary which build it was, on a tool distributed mainly as an image. | + +Both images also ran as root. CI now fails if any Dockerfile pins a version +other than the declared `rust-version`. + +No 0.6.0 or 0.7.0 image was ever published, and no `-cli` or `-ui` image since +0.5.0, because the release pipeline could +not complete: `build-macos-intel` targeted the `macos-13` runner image, retired +in December 2025, and hung for the full 24-hour limit on every tag. Fixing that +exposed a second failure that had been unreachable behind it — the workflow +never declared `permissions: contents: write`, so creating a release was refused +with a 403. Both are fixed. + +**Anyone running an image older than 0.8.0 is on 0.5.1 or earlier**, which +predates every security fix listed above. + +Still requiring a person: decide whether to publish a GHSA now that a fixed +version exists. ## If you are deciding whether to run this diff --git a/deployment_assets/GITHUB_ACTIONS.md b/deployment_assets/GITHUB_ACTIONS.md index 34737c0..aa1d323 100644 --- a/deployment_assets/GITHUB_ACTIONS.md +++ b/deployment_assets/GITHUB_ACTIONS.md @@ -179,6 +179,6 @@ The workflow: ## Related Documentation -- [bin/README.md](./bin/README.md) - Using pre-compiled binaries +- bin/README.md - Using pre-compiled binaries - [GitHub Actions Documentation](https://docs.github.com/en/actions) - [Rust Cross-Compilation Guide](https://rust-lang.github.io/rustup/cross-compilation.html) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 57d9b7e..77fd764 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -9,7 +9,7 @@ This directory contains detailed technical documentation for the Pangolin archit - **[Enums](./enums.md)**: Exhaustive list of system enumerations and their serialized values. ## 🔧 Interfaces & Logic -- **[System Traits](./traits.md)**: In-depth look at `CatalogStore` and `Signer` interfaces. +- **[System Traits](./catalog-store-trait.md)**: In-depth look at `CatalogStore` and `Signer` interfaces. - **[Branching & Merging](./branching.md)**: Operational details of the "Git-for-Data" versioning model. - **[Caching Strategy](./caching.md)**: multi-layered performance optimizations for metadata and cloud backends. diff --git a/docs/architecture/handlers.md b/docs/architecture/handlers.md index 56806d9..b1dabdd 100644 --- a/docs/architecture/handlers.md +++ b/docs/architecture/handlers.md @@ -10,7 +10,7 @@ This document lists the handler modules responsible for the API implementation, - `list_tables` / `create_table`: Table lifecycle. - `load_table` / `update_table` / `delete_table`: Table operations. - `report_metrics`: Metrics reporting. -- **Planned Refactor**: See [Iceberg Modularization Plan](../../planning/modularization_plan_iceberg.md). +- **Planned Refactor**: See Iceberg Modularization Plan. ## Tenant & Storage Management **Files**: @@ -58,4 +58,4 @@ This document lists the handler modules responsible for the API implementation, ## CLI Admin Handlers **File**: `pangolin_cli_admin/src/handlers.rs` (Refactor to `handlers/` in progress) -- **Planned Refactor**: See [CLI Modularization Plan](../../planning/modularization_plan_cli.md). +- **Planned Refactor**: See CLI Modularization Plan. diff --git a/docs/architecture/signer-trait.md b/docs/architecture/signer-trait.md index 339159f..a1f02ab 100644 --- a/docs/architecture/signer-trait.md +++ b/docs/architecture/signer-trait.md @@ -238,5 +238,5 @@ let creds = store.vend_credentials(&warehouse.storage_config, &prefix).await?; ## See Also - [CatalogStore Trait](./catalog-store-trait.md) -- [Credential Vending Guide](../guides/credential-vending.md) -- [Storage Configuration](../configuration/storage.md) +- [Credential Vending Guide](../features/security_vending.md) +- [Storage Configuration](./storage_and_connectivity.md) diff --git a/docs/architecture/storage_and_connectivity.md b/docs/architecture/storage_and_connectivity.md index 19b0841..6733082 100644 --- a/docs/architecture/storage_and_connectivity.md +++ b/docs/architecture/storage_and_connectivity.md @@ -28,7 +28,7 @@ graph TD ## Modular Storage Architecture -As of December 2025, all Pangolin storage backends have been refactored into a modular pattern. See the **[Backend Architecture Audit](../../planning/backend_architecture_audit.md)** for detailed status. +As of December 2025, all Pangolin storage backends have been refactored into a modular pattern. See the **Backend Architecture Audit** for detailed status. Each backend now resides in its own directory: - `pangolin_store/src/postgres/` diff --git a/docs/cli/admin-optimization-commands.md b/docs/cli/admin-optimization-commands.md index a1a5c77..589e376 100644 --- a/docs/cli/admin-optimization-commands.md +++ b/docs/cli/admin-optimization-commands.md @@ -270,4 +270,4 @@ Error: API Request Failed: Connection refused - [Admin CLI Overview](./admin.md) - [API Reference](../api/api_overview.md) -- [Performance Optimizations](../../planning/performance_optimizations_status.md) +- Performance Optimizations diff --git a/docs/cli/admin-service-users.md b/docs/cli/admin-service-users.md index 5f4ecbe..6dd3470 100644 --- a/docs/cli/admin-service-users.md +++ b/docs/cli/admin-service-users.md @@ -277,7 +277,7 @@ Use descriptive names that indicate: ## Related Documentation -- [Service Users API](../service_users.md) +- [Service Users API](../features/service_users.md) - [Authentication](../authentication.md) - [RBAC](../features/rbac.md) - [CLI Overview](./admin.md) diff --git a/docs/cli/admin.md b/docs/cli/admin.md index d682184..437c17d 100644 --- a/docs/cli/admin.md +++ b/docs/cli/admin.md @@ -112,7 +112,7 @@ Complete merge workflow for branch management. See [Merge Operations Guide](./ad ## Business Metadata & Governance -Manage business metadata and access requests. See [Business Metadata Guide](./admin-business-metadata.md) for details. +Manage business metadata and access requests. See [Business Metadata Guide](../features/business_catalog.md) for details. ### Commands - `delete-metadata --asset-id `: Delete business metadata diff --git a/docs/cli/docker-usage.md b/docs/cli/docker-usage.md index 2ffd895..558e029 100644 --- a/docs/cli/docker-usage.md +++ b/docs/cli/docker-usage.md @@ -239,4 +239,4 @@ curl -H "Authorization: Bearer $PANGOLIN_TOKEN" http://localhost:8080/api/v1/ten - [Admin CLI Reference](admin.md) - [User CLI Reference](user.md) - [Configuration Guide](configuration.md) -- [Binary Installation](../deployment_assets/bin/README.md) +- Binary Installation diff --git a/docs/cli/warehouse-management.md b/docs/cli/warehouse-management.md index 1e47247..0022587 100644 --- a/docs/cli/warehouse-management.md +++ b/docs/cli/warehouse-management.md @@ -307,7 +307,7 @@ pangolin-admin create-catalog prod-catalog-azure --warehouse production-azure ## Related Documentation - [Storage & Connectivity Architecture](../architecture/storage_and_connectivity.md) - Detailed multi-cloud architecture -- [PyIceberg Integration Guide](../../planning/pyiceberg_testing_guide.md) - Testing warehouses with PyIceberg +- PyIceberg Integration Guide - Testing warehouses with PyIceberg - [Authentication Guide](../architecture/authentication.md) - CLI authentication methods ## See Also diff --git a/docs/environment-variables.md b/docs/environment-variables.md index f510f2f..37e817a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -699,6 +699,6 @@ export PANGOLIN_NO_AUTH=true ## See Also -- [Deployment Guide](./deployment.md) -- [Docker Setup](./docker-setup.md) -- [Configuration Best Practices](./best-practices/configuration.md) +- [Deployment Guide](./getting-started/deployment.md) +- Docker Setup +- [Configuration Best Practices](./getting-started/configuration.md) diff --git a/docs/features/rbac.md b/docs/features/rbac.md index 7ed4546..20ac561 100644 --- a/docs/features/rbac.md +++ b/docs/features/rbac.md @@ -167,7 +167,7 @@ Authorization: Bearer ## Service Users -Service users are programmatic identities with API key authentication. See [Service Users](../service_users.md) for details. +Service users are programmatic identities with API key authentication. See [Service Users](./service_users.md) for details. **Key Points**: - Service users inherit the same RBAC system @@ -318,7 +318,7 @@ POST /api/v1/permissions ## Related Documentation -- [Service Users](../service_users.md) - API key authentication +- [Service Users](./service_users.md) - API key authentication - [Authentication](../authentication.md) - User authentication - [Audit Logs](./audit_logs.md) - Access monitoring - [Permissions System](../permissions.md) - Permission details diff --git a/docs/features/time_travel.md b/docs/features/time_travel.md index 4daabab..9784c2b 100644 --- a/docs/features/time_travel.md +++ b/docs/features/time_travel.md @@ -304,5 +304,5 @@ if len(snapshots) > 1: ## Related Documentation - [Branch Management](./branch_management.md) - Git-like branching for catalogs -- [Merge Conflicts](../merge_conflicts.md) - Merging branches +- [Merge Conflicts](merge_conflicts.md) - Merging branches - [PyIceberg Testing](./pyiceberg_testing.md) - PyIceberg integration diff --git a/docs/known-issues/README.md b/docs/known-issues/README.md index 1ff86db..8f65e5e 100644 --- a/docs/known-issues/README.md +++ b/docs/known-issues/README.md @@ -3,8 +3,13 @@ This section documents verified issues, limitations, and architectural quirks present in the current release. -## v0.4.0 +## Resolved * [SQL Backend Token Listing (SQLite/Postgres)](./token_listing_sqlite_join.md) - * **Description**: Active token lists may be empty for Root users or ephemeral accounts when using SQL backends due to a strict `JOIN` dependency. - * **Status**: Identified. Fix proposed for v0.5.x. + * **Description**: Active token lists were empty for Root or ephemeral accounts on SQL backends, because the query inner-joined `active_tokens` to `users` and an ephemeral root has no row there. + * **Status**: **Fixed.** Both backends now filter `tenant_id` directly off `active_tokens`, matching what the memory and MongoDB backends always did. Verified by the cross-backend parity suite, which asserts identical behaviour on all four. + +## Current + +No verified issues are open against 0.8.0 beyond the limitations recorded in +[STATUS.md](../../STATUS.md), which is the authoritative list. diff --git a/docs/known-issues/token_listing_sqlite_join.md b/docs/known-issues/token_listing_sqlite_join.md index 1635f2b..7327288 100644 --- a/docs/known-issues/token_listing_sqlite_join.md +++ b/docs/known-issues/token_listing_sqlite_join.md @@ -1,7 +1,14 @@ # Known Issue: SQL Backend Token Listing for Root/Ephemeral Users -**Affected Versions**: v0.4.0 and prior +> **Resolved in 0.7.0.** Both SQL backends now select `tenant_id` directly from +> `active_tokens` instead of joining to `users`, which is what the memory and +> MongoDB backends always did. The cross-backend parity suite asserts all four +> behave identically, so this cannot silently return. +> +> Kept as a record of the defect and its cause. + +**Affected Versions**: v0.4.0 through v0.6.0 **Affected Backends**: SQLite, PostgreSQL **Unaffected Backends**: Memory, MongoDB diff --git a/docs/operations/backend-parity.md b/docs/operations/backend-parity.md index 8bd1773..3fba456 100644 --- a/docs/operations/backend-parity.md +++ b/docs/operations/backend-parity.md @@ -103,7 +103,11 @@ authenticate on MongoDB at all. Remaining gaps: ``` They use different ports so both can run at once. -* Multi-statement operations other than `delete_catalog` are still not atomic. +* Multi-statement operations other than `delete_catalog` are still not atomic on + MongoDB. PostgreSQL and SQLite wrap branch-creation-by-copy from 0.8.0; + MongoDB has no atomic version, so the API takes a sequential fallback, logs + that a partial failure will leave the branch incomplete, and returns a `500` + naming it rather than a misleading `200`. ## What the first live run found diff --git a/docs/upgrading/0.7-to-0.8.md b/docs/upgrading/0.7-to-0.8.md new file mode 100644 index 0000000..8645293 --- /dev/null +++ b/docs/upgrading/0.7-to-0.8.md @@ -0,0 +1,129 @@ +# Upgrading to 0.8.0 + +0.8.0 is a security release. Nothing here requires a data migration, but four +changes need a decision before you deploy, and one can fail loudly on startup if +your MongoDB holds data the previous version allowed. + +If you are coming from 0.6.x or earlier, read +[0.6-to-0.7.md](0.6-to-0.7.md) first. + +> **If you run a container image, you are almost certainly not on 0.7.0.** No +> 0.6.0 or 0.7.0 image was ever published — the release pipeline could not +> complete — so `alexmerced/pangolin-api` went from 0.5.1 straight to 0.8.0. +> Treat this as an upgrade from 0.5.1 and read the 0.6 and 0.7 notes too. + +## 1. Minimum Rust version is now 1.94 + +Only relevant if you build from source. It was raised from 1.92 deliberately, to +pick up AWS SDK releases carrying fixed `aws-lc-sys` and `rustls-webpki` — that +is what cleared two high-severity certificate-validation advisories on the path +Pangolin uses to reach S3, Azure Blob and GCS. + +The published binaries and images are unaffected. + +## 2. Set `PANGOLIN_ENCRYPTION_KEY` + +Warehouse cloud credentials can now be encrypted at rest. **This is off unless +you set a key**, and the server logs a warning at startup while it is unset. + +```bash +openssl rand -base64 32 +``` + +Three things to know before you turn it on: + +- **The key is not in a database dump.** Back it up separately, or a restore + produces a working catalog whose every warehouse credential is unreadable. +- **Existing warehouses stay in plaintext** until something rewrites them. + Reads tolerate both, so the upgrade is not an outage. See + [operations/encryption.md](../operations/encryption.md) for how to find and + re-seal them. +- **Losing the key is unrecoverable.** Treat it like `PANGOLIN_JWT_SECRET`. + +If you run more than one replica, **every replica needs the same key**, or a +warehouse sealed by one is unreadable by the others. + +## 3. MongoDB now enforces uniqueness — this can fail on existing data + +MongoDB previously accepted two catalogs with the same name in one tenant and +returned an arbitrary one on lookup. 0.8.0 creates the unique indexes the SQL +backends have always had, covering catalogs, warehouses, branches, tags, and one +business-metadata record per asset. + +**If your database already contains duplicates, index creation fails.** Startup +continues — refusing to boot over an index would turn a data problem into an +outage — but the log carries an error naming the collection, and that uniqueness +is *not* enforced until you deduplicate and restart. + +Check before upgrading: + +```javascript +// Duplicate catalog names within a tenant +db.catalogs.aggregate([ + { $group: { _id: { tenant_id: "$tenant_id", name: "$name" }, n: { $sum: 1 } } }, + { $match: { n: { $gt: 1 } } } +]) +``` + +Repeat for `warehouses`, `branches` and `tags` if that returns anything. + +## 4. Authentication endpoints are now rate limited + +`/api/v1/users/login`, `/api/v1/tokens` and the OAuth callback allow 10 failed +attempts per minute, counted per source address **and** separately per account. + +Tune with `PANGOLIN_AUTH_RATE_LIMIT` and `PANGOLIN_AUTH_RATE_WINDOW_SECS`; `0` +disables it. + +**If you run behind a proxy**, set `PANGOLIN_TRUST_FORWARDED_FOR=true` so the +limit keys on the real client rather than your load balancer — otherwise every +request appears to come from one address and legitimate users throttle each +other. **Only set it behind a proxy that overwrites `X-Forwarded-For`**: trusting +it otherwise lets a caller set a fresh value per request and bypass the limit +entirely. + +The counters are in-process, so with N replicas the effective limit is N times +the configured one. + +## 5. OIDC applies automatically + +If you use Google, Microsoft or Okta, logins now go through real OIDC — PKCE, +`id_token` signature validation against the provider's JWKS, and +`iss`/`aud`/`exp`/`nonce` checks. No configuration change is needed; the issuer +is derived from the settings you already have. + +For a self-hosted IdP (Keycloak, Auth0, a private Okta), set +`PANGOLIN__ISSUER`. + +**GitHub is unaffected and cannot be**: it issues no `id_token` and publishes no +JWKS, so its logins still rest on the userinfo endpoint. Set +`PANGOLIN_OIDC_REQUIRE=true` to refuse any provider that cannot be validated — +this will reject GitHub logins, which is the point. + +OAuth requires session affinity across replicas: the PKCE verifier is held in +process, deliberately not in the `state` parameter, because `state` travels +through the browser alongside the authorization code. + +## Behaviour changes worth knowing + +- **Creating a branch by copy now fails loudly.** It previously returned `200` + even when copying the assets failed, leaving an empty branch reported as + ready. It is now transactional on PostgreSQL and SQLite; on MongoDB the API + falls back to sequential statements and returns a `500` naming the branch. +- **The revocation sweep now runs.** It was defined and never started, so + `revoked_tokens` grew for the life of the deployment. Expect a one-off delete + of accumulated expired records on first run. +- **New Iceberg endpoints:** `registerTable`, `listViews`, `viewExists`, + `dropView`. `commitTransaction` remains deliberately unimplemented. + +## Checklist + +1. Read [SECURITY.md](../../SECURITY.md) and rotate `PANGOLIN_JWT_SECRET` if + coming from 0.6.x or earlier. +2. Generate and store `PANGOLIN_ENCRYPTION_KEY`; back it up where you keep + break-glass secrets. +3. On MongoDB, check for duplicates using the query above. +4. Set `PANGOLIN_TRUST_FORWARDED_FOR` if you run behind a proxy. +5. Take a backup, and confirm you can restore it — + `scripts/backup_restore_drill.sh` does the whole cycle. +6. Deploy, then check the startup log for encryption and index warnings. diff --git a/pangolin/Dockerfile.tools b/pangolin/Dockerfile.tools index b538280..8322493 100644 --- a/pangolin/Dockerfile.tools +++ b/pangolin/Dockerfile.tools @@ -1,5 +1,9 @@ # Build Stage -FROM rust:1.88-slim-bookworm as builder +# Kept in step with `rust-version` in Cargo.toml, which the `msrv` CI job +# verifies. A-36 raised the API image and missed this one, so the CLI image +# stayed on 1.88 and failed to build the moment the workspace MSRV moved past +# it - which is what happened publishing 0.8.0. +FROM rust:1.94-slim-bookworm AS builder WORKDIR /usr/src/pangolin @@ -17,15 +21,33 @@ COPY pangolin_cli_common ./pangolin_cli_common COPY pangolin_cli_admin ./pangolin_cli_admin COPY pangolin_cli_user ./pangolin_cli_user -# Build CLI binaries -RUN cargo build --release --bin pangolin-admin -RUN cargo build --release --bin pangolin-user +# Build CLI binaries. +# +# One invocation, not two: the second `cargo build` re-resolved and re-linked +# the shared dependency graph for no benefit. `--locked` makes the build fail +# rather than silently resolve a different dependency set than Cargo.lock +# records - a published binary should be reproducible from the committed lock. +RUN cargo build --release --locked --bin pangolin-admin --bin pangolin-user # Runtime Stage FROM debian:bookworm-slim -# Install OpenSSL and CA certificates (needed for remote connections) -RUN apt-get update && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* +LABEL org.opencontainers.image.title="Pangolin CLI" \ + org.opencontainers.image.description="Admin and user command-line clients for the Pangolin catalog" \ + org.opencontainers.image.source="https://github.com/AlexMercedCoder/pangolin" \ + org.opencontainers.image.licenses="MIT" + +# libssl3, not libssl-dev: the runtime needs the shared library, not headers +# and static archives. A-36 corrected this in the API image and missed this +# one, so the CLI image was still shipping the development package. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Run as an unprivileged user (A-35). These are interactive clients that read +# credentials from a mounted config; there is no reason for them to be root. +RUN groupadd --system --gid 10001 pangolin \ + && useradd --system --uid 10001 --gid pangolin --home-dir /app --shell /usr/sbin/nologin pangolin WORKDIR /app @@ -33,5 +55,7 @@ WORKDIR /app COPY --from=builder /usr/src/pangolin/target/release/pangolin-admin /usr/local/bin/pangolin-admin COPY --from=builder /usr/src/pangolin/target/release/pangolin-user /usr/local/bin/pangolin-user +USER 10001:10001 + # Default command (help) CMD ["echo", "Available commands: pangolin-admin, pangolin-user"] diff --git a/pangolin/pangolin_cli_admin/src/main.rs b/pangolin/pangolin_cli_admin/src/main.rs index a1b1a90..0ed9665 100644 --- a/pangolin/pangolin_cli_admin/src/main.rs +++ b/pangolin/pangolin_cli_admin/src/main.rs @@ -10,6 +10,10 @@ use rustyline::Editor; #[derive(Parser, Debug)] #[command(name = "pangolin-admin")] +// `version` reads CARGO_PKG_VERSION. Without it there was no way to ask a +// binary which build it was - which matters most for the container image, +// where the tag is the only other clue and `latest` tells you nothing. +#[command(version)] struct Args { #[arg(long, env = "PANGOLIN_URL")] url: Option, diff --git a/pangolin/pangolin_cli_user/src/main.rs b/pangolin/pangolin_cli_user/src/main.rs index 7de4395..135689a 100644 --- a/pangolin/pangolin_cli_user/src/main.rs +++ b/pangolin/pangolin_cli_user/src/main.rs @@ -10,6 +10,10 @@ use rustyline::Editor; #[derive(Parser, Debug)] #[command(name = "pangolin-user")] +// `version` reads CARGO_PKG_VERSION. Without it there was no way to ask a +// binary which build it was - which matters most for the container image, +// where the tag is the only other clue and `latest` tells you nothing. +#[command(version)] struct Args { #[arg(long, env = "PANGOLIN_URL")] url: Option, diff --git a/pangolin_docs.md b/pangolin_docs.md index bb3388a..e98c5d5 100644 --- a/pangolin_docs.md +++ b/pangolin_docs.md @@ -1,3 +1,14 @@ +` marker naming + the file it came from. + + Relative links inside are relative to the *original* file's location and do + not resolve here. Edit the source file, not this one. +--> + # Documentation Update Summary - Credential Vending @@ -764,10 +775,10 @@ Welcome to the comprehensive documentation for **Pangolin**, the cloud-native Ap *Quickest path from zero to a running lakehouse.* - **[Onboarding Index](./getting-started/README.md)** - **Start Here!** -- **[Installation Guide](./getting-started/getting_started.md)** - Run Pangolin in 5 minutes. -- **[Evaluating Pangolin](./getting-started/evaluating-pangolin.md)** - Rapid local testing with `NO_AUTH` mode. +- **[Installation Guide](docs/getting-started/getting_started.md)** - Run Pangolin in 5 minutes. +- **[Evaluating Pangolin](docs/getting-started/evaluating-pangolin.md)** - Rapid local testing with `NO_AUTH` mode. - **[Deployment Guide](./getting-started/deployment.md)** - Local, Docker, and Production setup. -- **[Environment Variables](./getting-started/env_vars.md)** - Complete system configuration reference. +- **[Environment Variables](docs/getting-started/env_vars.md)** - Complete system configuration reference. --- @@ -777,8 +788,8 @@ Welcome to the comprehensive documentation for **Pangolin**, the cloud-native Ap - **[Infrastructure Features](./features/README.md)** - Index of all platform capabilities. - **[Warehouse Management](./warehouse/README.md)** - Configuring S3, Azure, and GCS storage. - **[Metadata Backends](./backend_storage/README.md)** - Memory, Postgres, MongoDB, and SQLite. -- **[Asset Management](./features/asset_management.md)** - Tables, Views, and CRUD operations. -- **[Federated Catalogs](./features/federated_catalogs.md)** - Proxying external REST catalogs. +- **[Asset Management](docs/features/asset_management.md)** - Tables, Views, and CRUD operations. +- **[Federated Catalogs](docs/features/federated_catalogs.md)** - Proxying external REST catalogs. - **[Known Issues](./known-issues/README.md)** - Documented limitations and active bugs (e.g., SQL backend quirks). --- @@ -786,22 +797,22 @@ Welcome to the comprehensive documentation for **Pangolin**, the cloud-native Ap ## ⚖️ 3. Governance & Security *Multi-tenancy, RBAC, and auditing.* -- **[Security Concepts](./features/security_vending.md)** - Identity and Credential Vending principles. -- **[Credential Vending (IAM Roles)](./features/iam_roles.md)** - Scoped cloud access (STS, SAS, Downscoped). +- **[Security Concepts](docs/features/security_vending.md)** - Identity and Credential Vending principles. +- **[Credential Vending (IAM Roles)](docs/features/iam_roles.md)** - Scoped cloud access (STS, SAS, Downscoped). - **[Permission System](./permissions.md)** - Understanding RBAC and granular grants. - **[Service Users](./features/service_users.md)** - Programmatic access and API key management. -- **[Audit Logging](./features/audit_logs.md)** - Global action tracking and compliance. +- **[Audit Logging](docs/features/audit_logs.md)** - Global action tracking and compliance. --- ## 🧪 4. Data Life Cycle *Git-for-Data and maintenance workflows.* -- **[Branch Management](./features/branch_management.md)** - Working with isolated data environments. -- **[Merge Operations](./features/merge_operations.md)** - The 3-way merge workflow. -- **[Merge Conflicts](./features/merge_conflicts.md)** - Theory and resolution strategies. -- **[Business Metadata & Discovery](./features/business_catalog.md)** - Search, tags, and access requests. -- **[Maintenance Utilities](./features/maintenance.md)** - Snapshot expiration and compaction. +- **[Branch Management](docs/features/branch_management.md)** - Working with isolated data environments. +- **[Merge Operations](docs/features/merge_operations.md)** - The 3-way merge workflow. +- **[Merge Conflicts](docs/features/merge_conflicts.md)** - Theory and resolution strategies. +- **[Business Metadata & Discovery](docs/features/business_catalog.md)** - Search, tags, and access requests. +- **[Maintenance Utilities](docs/features/maintenance.md)** - Snapshot expiration and compaction. --- @@ -821,8 +832,8 @@ Welcome to the comprehensive documentation for **Pangolin**, the cloud-native Ap - **[Architecture Overview](./architecture/README.md)** - System design and component interaction. - **[Technical Logic Deep-Dive](./architecture/README.md)** - Caching, Branching (Git-for-Data), and Trait details. -- **[Data Models](./architecture/models.md)** - Understanding the internal schema. -- **[CatalogStore Trait](./architecture/catalog-store-trait.md)** - Extending Pangolin storage. +- **[Data Models](docs/architecture/models.md)** - Understanding the internal schema. +- **[CatalogStore Trait](docs/architecture/catalog-store-trait.md)** - Extending Pangolin storage. - **[Developer Utilities](./utilities/README.md)** - Tools for contributors (e.g. OpenAPI generation). --- @@ -832,7 +843,7 @@ Welcome to the comprehensive documentation for **Pangolin**, the cloud-native Ap - **[Best Practices Index](./best-practices/README.md)** - Complete guide to operating Pangolin. - **[Deployment & Security](./best-practices/deployment.md)** - Production checklists. -- **[Scalability](./best-practices/scalability.md)** - Tuning for high performance. +- **[Scalability](docs/best-practices/scalability.md)** - Tuning for high performance. - **[Iceberg Tuning](./best-practices/iceberg.md)** - Optimizing table layout and compaction. --- @@ -855,7 +866,7 @@ Pangolin implements the [Apache Iceberg REST Catalog specification](https://gith ## Core API -### [API Overview](api_overview.md) +### [API Overview](docs/api/api_overview.md) Complete REST API reference covering: - Namespace operations - Table operations @@ -949,7 +960,7 @@ Pangolin provides an interactive Swagger UI for live API exploration: | Document | Description | |----------|-------------| -| [api_overview.md](api_overview.md) | Complete REST API reference | +| [api_overview.md](docs/api/api_overview.md) | Complete REST API reference | | [authentication.md](authentication.md) | Authentication methods and setup | ## Quick Examples @@ -1197,7 +1208,7 @@ The API is protected by authentication middleware that: ## S3 Credential Vending -For direct S3 access (e.g., from engines like Spark or Trino), Pangolin provides a credential vending mechanism. See [Security & Vending](../features/security_vending.md) for details. +For direct S3 access (e.g., from engines like Spark or Trino), Pangolin provides a credential vending mechanism. See [Security & Vending](docs/features/security_vending.md) for details. --- @@ -1881,7 +1892,7 @@ echo "✅ Environment created successfully!" - [OpenAPI Specification](./openapi.yaml) - Complete API schema - [Apache Iceberg REST Spec](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml) - Iceberg endpoints - [Authentication Guide](../authentication.md) - Authentication setup -- [Getting Started](../getting-started/getting_started.md) - Quick start guide +- [Getting Started](docs/getting-started/getting_started.md) - Quick start guide --- @@ -2318,19 +2329,19 @@ curl -X POST http://localhost:8080/api/v1/bulk/assets/delete \ This directory contains detailed technical documentation for the Pangolin architecture. ## 🏗️ Core Structure -- **[High-Level Architecture](./architecture.md)**: Overall system design, component interaction, and multi-tenant isolation. -- **[API Handlers](./handlers.md)**: Map of API endpoints categorized by functional domain (Iceberg, Versioning, Admin). -- **[Models](./models.md)**: Comprehensive guide to core system structs (Tenant, Asset, Merge, User). -- **[Enums](./enums.md)**: Exhaustive list of system enumerations and their serialized values. +- **[High-Level Architecture](docs/architecture/architecture.md)**: Overall system design, component interaction, and multi-tenant isolation. +- **[API Handlers](docs/architecture/handlers.md)**: Map of API endpoints categorized by functional domain (Iceberg, Versioning, Admin). +- **[Models](docs/architecture/models.md)**: Comprehensive guide to core system structs (Tenant, Asset, Merge, User). +- **[Enums](docs/architecture/enums.md)**: Exhaustive list of system enumerations and their serialized values. ## 🔧 Interfaces & Logic - **[System Traits](./traits.md)**: In-depth look at `CatalogStore` and `Signer` interfaces. - **[Branching & Merging](./branching.md)**: Operational details of the "Git-for-Data" versioning model. -- **[Caching Strategy](./caching.md)**: multi-layered performance optimizations for metadata and cloud backends. +- **[Caching Strategy](docs/architecture/caching.md)**: multi-layered performance optimizations for metadata and cloud backends. ## 🔐 Security & Operations - **[Authentication](./authentication.md)**: Deep dive into JWT, Service User API Keys, and RBAC. -- **[Storage & Connectivity](./storage_and_connectivity.md)**: Cloud connectivity, modular store structure, and credential vending. +- **[Storage & Connectivity](docs/architecture/storage_and_connectivity.md)**: Cloud connectivity, modular store structure, and credential vending. - **[Dependencies](./dependencies.md)**: Final list of technology stack and library versions. --- @@ -3544,7 +3555,7 @@ The entire backend stack is fully asynchronous, built on `tokio` and `axum`. - **Background Tasks**: Tasks like `cleanup_expired_tokens` run in detached Tokio tasks to avoid blocking request paths. ## 4. Caching Layers -Performance is further augmented by a multi-tier caching strategy explained in [Caching Architecture](./caching.md). +Performance is further augmented by a multi-tier caching strategy explained in [Caching Architecture](docs/architecture/caching.md). - **Level 1**: Wrapper Cache (`CachedCatalogStore`) - Configuration data. - **Level 2**: Metadata Cache (`moka`) - Iceberg Manifest bytes. - **Level 3**: Connection Pools (`ObjectStoreCache`, DB Pools). @@ -3793,7 +3804,7 @@ let creds = store.vend_credentials(&warehouse.storage_config, &prefix).await?; ## See Also -- [CatalogStore Trait](./catalog-store-trait.md) +- [CatalogStore Trait](docs/architecture/catalog-store-trait.md) - [Credential Vending Guide](../guides/credential-vending.md) - [Storage Configuration](../configuration/storage.md) @@ -4338,7 +4349,7 @@ catalog = load_catalog( ### Option B: Credential Vending (Recommended) Pangolin automatically vends S3/Cloud credentials if the Warehouse is configured with a role. You only need to provide the Pangolin token. -- See [Credential Vending](./features/security_vending.md) for warehouse setup. +- See [Credential Vending](docs/features/security_vending.md) for warehouse setup. --- @@ -4446,11 +4457,11 @@ DATABASE_URL=mongodb://user:password@localhost:27017/pangolin ## Next Steps -- [In-Memory Setup Guide](memory.md) -- [SQLite Setup Guide](sqlite.md) +- [In-Memory Setup Guide](docs/backend_storage/memory.md) +- [SQLite Setup Guide](docs/backend_storage/sqlite.md) - [PostgreSQL Setup Guide](postgresql.md) - [MongoDB Setup Guide](mongodb.md) -- [Detailed Comparison](comparison.md) +- [Detailed Comparison](docs/backend_storage/comparison.md) ## Migration Between Backends @@ -4726,7 +4737,7 @@ All three backends are production-ready and fully tested. Your choice should be - [PostgreSQL Setup](postgresql.md) - [MongoDB Setup](mongodb.md) -- [SQLite Setup](sqlite.md) +- [SQLite Setup](docs/backend_storage/sqlite.md) - [Warehouse Storage](../warehouse/README.md) @@ -5107,17 +5118,17 @@ curl -X POST http://localhost:8080/api/v1/warehouses \ ## Additional Resources - [DashMap Documentation](https://docs.rs/dashmap/) -- [Backend Storage Comparison](comparison.md) +- [Backend Storage Comparison](docs/backend_storage/comparison.md) - [PostgreSQL Backend](postgresql.md) - [MongoDB Backend](mongodb.md) -- [SQLite Backend](sqlite.md) +- [SQLite Backend](docs/backend_storage/sqlite.md) ## Next Steps - [PostgreSQL Backend](postgresql.md) - For production deployments - [MongoDB Backend](mongodb.md) - For cloud-native deployments -- [SQLite Backend](sqlite.md) - For embedded deployments -- [Backend Comparison](comparison.md) - Choose the right backend +- [SQLite Backend](docs/backend_storage/sqlite.md) - For embedded deployments +- [Backend Comparison](docs/backend_storage/comparison.md) - Choose the right backend --- @@ -5414,8 +5425,8 @@ db.runCommand({ compact: 'catalogs' }) ## Next Steps - [PostgreSQL Backend](postgresql.md) -- [SQLite Backend](sqlite.md) -- [Backend Comparison](comparison.md) +- [SQLite Backend](docs/backend_storage/sqlite.md) +- [Backend Comparison](docs/backend_storage/comparison.md) - [Warehouse Storage](../warehouse/README.md) @@ -5783,8 +5794,8 @@ Custom migration script required. Contact support or see migration guide. ## Next Steps - [MongoDB Backend](mongodb.md) -- [SQLite Backend](sqlite.md) -- [Backend Comparison](comparison.md) +- [SQLite Backend](docs/backend_storage/sqlite.md) +- [Backend Comparison](docs/backend_storage/comparison.md) - [Warehouse Storage](../warehouse/README.md) @@ -6118,7 +6129,7 @@ rm dev.db - [PostgreSQL Backend](postgresql.md) - [MongoDB Backend](mongodb.md) -- [Backend Comparison](comparison.md) +- [Backend Comparison](docs/backend_storage/comparison.md) - [Warehouse Storage](../warehouse/README.md) @@ -6134,7 +6145,7 @@ Comprehensive guides for deploying, operating, and optimizing Pangolin in produc ### Operations - **[Deployment](./deployment.md)** - Production deployment strategies, Docker, Kubernetes, HA setup -- **[Scalability](./scalability.md)** - Scaling API servers, databases, multi-tenancy, performance optimization +- **[Scalability](docs/best-practices/scalability.md)** - Scaling API servers, databases, multi-tenancy, performance optimization - **[Security](./security.md)** - Authentication, encryption, audit logging, compliance ### Data Management @@ -6144,7 +6155,7 @@ Comprehensive guides for deploying, operating, and optimizing Pangolin in produc ### Technical - **[Apache Iceberg](./iceberg.md)** - Table design, partitioning, schema evolution, performance tuning -- **[Generic Assets](./generic-assets.md)** - Managing ML models, files, media, and other artifacts +- **[Generic Assets](docs/best-practices/generic-assets.md)** - Managing ML models, files, media, and other artifacts ## Quick Reference @@ -6654,10 +6665,10 @@ ORDER BY week DESC; ## Additional Resources -- [Branch Management Guide](../features/branch_management.md) -- [Merge Operations](../features/merge_operations.md) -- [Conflict Resolution](../features/merge_conflicts.md) -- [Git Operations (PyPangolin)](../../pypangolin/docs/git_operations.md) +- [Branch Management Guide](docs/features/branch_management.md) +- [Merge Operations](docs/features/merge_operations.md) +- [Conflict Resolution](docs/features/merge_conflicts.md) +- [Git Operations (PyPangolin)](pypangolin/docs/git_operations.md) --- @@ -7099,9 +7110,9 @@ CREATE INDEX idx_audit_timestamp ON audit_logs(timestamp); ## Additional Resources -- [Scalability Best Practices](./scalability.md) +- [Scalability Best Practices](docs/best-practices/scalability.md) - [Security Best Practices](./security.md) -- [Monitoring Guide](../features/audit_logs.md) +- [Monitoring Guide](docs/features/audit_logs.md) --- @@ -7584,10 +7595,10 @@ pangolin-admin grant-permission \ ## Additional Resources -- [Generic Assets Guide](../features/generic_assets.md) -- [PyPangolin File Assets](../../pypangolin/docs/csv.md) -- [PyPangolin ML Models](../../pypangolin/docs/other.md) -- [Lance Format](../../pypangolin/docs/lance.md) +- [Generic Assets Guide](docs/features/generic_assets.md) +- [PyPangolin File Assets](pypangolin/docs/csv.md) +- [PyPangolin ML Models](pypangolin/docs/other.md) +- [Lance Format](pypangolin/docs/lance.md) --- @@ -7971,9 +7982,9 @@ source_df.writeTo("analytics.new_iceberg_table").create() ## Additional Resources -- [PyIceberg Integration](../features/pyiceberg_testing.md) -- [Table Formats Guide](../features/table_formats.md) -- [Maintenance Operations](../features/maintenance.md) +- [PyIceberg Integration](docs/features/pyiceberg_testing.md) +- [Table Formats Guide](docs/features/table_formats.md) +- [Maintenance Operations](docs/features/maintenance.md) - [PyPangolin Iceberg Guide](../../pypangolin/docs/iceberg.md) @@ -8528,9 +8539,9 @@ def generate_metadata_dashboard(): ## Additional Resources -- [Business Catalog Guide](../features/business_catalog.md) -- [Tag Management](../features/tag_management.md) -- [Data Discovery](../ui/discovery_governance.md) +- [Business Catalog Guide](docs/features/business_catalog.md) +- [Tag Management](docs/features/tag_management.md) +- [Data Discovery](docs/ui/discovery_governance.md) --- @@ -8995,8 +9006,8 @@ pangolin-admin list-audit-events \ ## Additional Resources - [Security Best Practices](./security.md) -- [RBAC Documentation](../features/rbac.md) -- [Audit Logging](../features/audit_logs.md) +- [RBAC Documentation](docs/features/rbac.md) +- [Audit Logging](docs/features/audit_logs.md) --- @@ -9430,7 +9441,7 @@ aws s3 cp s3://hot-bucket/old-data/ s3://archive-bucket/old-data/ \ ## Additional Resources - [Deployment Best Practices](./deployment.md) -- [Performance Tuning Guide](../features/maintenance.md) +- [Performance Tuning Guide](docs/features/maintenance.md) - [Database Optimization](../backend_storage/README.md) @@ -9894,7 +9905,7 @@ pangolin-admin list-audit-events \ ## Additional Resources - [Permissions Management Best Practices](./permissions.md) -- [Audit Logging Guide](../features/audit_logs.md) +- [Audit Logging Guide](docs/features/audit_logs.md) - [Deployment Security](./deployment.md) @@ -9950,33 +9961,33 @@ pangolin-admin create-catalog my-catalog --warehouse my-warehouse ### Admin Tool (`pangolin-admin`) - **[Admin Overview](./admin.md)** - Complete admin tool reference -- **[Tenant Management](./admin-tenants.md)** - Creating and managing tenants -- **[User Management](./admin-users.md)** - User creation and administration -- **[Warehouse Management](./admin-warehouses.md)** - Storage backend configuration -- **[Catalog Management](./admin-catalogs.md)** - Local catalog operations -- **[Federated Catalogs](./admin-federated-catalogs.md)** - External catalog integration -- **[Permission Management](./admin-permissions.md)** - RBAC and access control -- **[Service Users](./admin-service-users.md)** - API keys and service accounts -- **[Token Management](./admin-token-management.md)** - User token operations -- **[Audit Logging](./admin-audit-logging.md)** - Viewing and analyzing audit events -- **[Merge Operations](./admin-merge-operations.md)** - Managing branch merges -- **[Update Operations](./admin-update-operations.md)** - Updating resources -- **[Metadata Management](./admin-metadata.md)** - Business metadata operations -- **[Optimization Commands](./admin-optimization-commands.md)** - Performance and maintenance +- **[Tenant Management](docs/cli/admin-tenants.md)** - Creating and managing tenants +- **[User Management](docs/cli/admin-users.md)** - User creation and administration +- **[Warehouse Management](docs/cli/admin-warehouses.md)** - Storage backend configuration +- **[Catalog Management](docs/cli/admin-catalogs.md)** - Local catalog operations +- **[Federated Catalogs](docs/cli/admin-federated-catalogs.md)** - External catalog integration +- **[Permission Management](docs/cli/admin-permissions.md)** - RBAC and access control +- **[Service Users](docs/cli/admin-service-users.md)** - API keys and service accounts +- **[Token Management](docs/cli/admin-token-management.md)** - User token operations +- **[Audit Logging](docs/cli/admin-audit-logging.md)** - Viewing and analyzing audit events +- **[Merge Operations](docs/cli/admin-merge-operations.md)** - Managing branch merges +- **[Update Operations](docs/cli/admin-update-operations.md)** - Updating resources +- **[Metadata Management](docs/cli/admin-metadata.md)** - Business metadata operations +- **[Optimization Commands](docs/cli/admin-optimization-commands.md)** - Performance and maintenance ### User Tool (`pangolin-user`) - **[User Overview](./user.md)** - Complete user tool reference -- **[Branch Management](./user-branches.md)** - Creating and managing branches -- **[Tag Management](./user-tags.md)** - Versioning with tags -- **[Discovery](./user-discovery.md)** - Data discovery and search -- **[Access Requests](./user-access.md)** - Requesting permissions -- **[Token Management](./user-tokens.md)** - Personal token management +- **[Branch Management](docs/cli/user-branches.md)** - Creating and managing branches +- **[Tag Management](docs/cli/user-tags.md)** - Versioning with tags +- **[Discovery](docs/cli/user-discovery.md)** - Data discovery and search +- **[Access Requests](docs/cli/user-access.md)** - Requesting permissions +- **[Token Management](docs/cli/user-tokens.md)** - Personal token management ### Configuration & Setup - **[Overview](./overview.md)** - CLI tools overview - **[Configuration](./configuration.md)** - CLI configuration files -- **[Docker Usage](./docker-usage.md)** - Running CLI in Docker -- **[Warehouse Management](./warehouse-management.md)** - Multi-cloud warehouse setup +- **[Docker Usage](docs/cli/docker-usage.md)** - Running CLI in Docker +- **[Warehouse Management](docs/cli/warehouse-management.md)** - Multi-cloud warehouse setup ## Common Tasks @@ -11338,7 +11349,7 @@ Error: API Request Failed: Connection refused ## See Also - [Admin CLI Overview](./admin.md) -- [API Reference](../api/api_overview.md) +- [API Reference](docs/api/api_overview.md) - [Performance Optimizations](../../planning/performance_optimizations_status.md) @@ -11740,7 +11751,7 @@ Use descriptive names that indicate: - [Service Users API](../service_users.md) - [Authentication](../authentication.md) -- [RBAC](../features/rbac.md) +- [RBAC](docs/features/rbac.md) - [CLI Overview](./admin.md) @@ -12586,14 +12597,14 @@ pangolin-admin login --username user --password pass123 --tenant-id $TENANT_ID - `revoke-permission `: Revoke permission from a role. ### Metadata -See [Metadata Management Guide](./admin-metadata.md) for detailed attribution and explorer commands. +See [Metadata Management Guide](docs/cli/admin-metadata.md) for detailed attribution and explorer commands. - `get-metadata --entity-type --entity-id `: Get entity properties. - `set-metadata --entity-type --entity-id `: Set entity properties. - `list-namespace-tree `: Browse catalog structure. ## Update Operations -Update existing resources. See [Update Operations Guide](./admin-update-operations.md) for details. +Update existing resources. See [Update Operations Guide](docs/cli/admin-update-operations.md) for details. ### Commands - `update-tenant --id --name `: Update tenant properties @@ -12603,7 +12614,7 @@ Update existing resources. See [Update Operations Guide](./admin-update-operatio ## Token Management -Manage authentication tokens for security. See [Token Management Guide](./admin-token-management.md) for details. +Manage authentication tokens for security. See [Token Management Guide](docs/cli/admin-token-management.md) for details. ### Commands - `revoke-token`: Revoke your own token (logout) @@ -12611,7 +12622,7 @@ Manage authentication tokens for security. See [Token Management Guide](./admin- ## Merge Operations -Complete merge workflow for branch management. See [Merge Operations Guide](./admin-merge-operations.md) for details. +Complete merge workflow for branch management. See [Merge Operations Guide](docs/cli/admin-merge-operations.md) for details. ### Commands - `list-merge-operations`: List all merge operations @@ -13745,7 +13756,7 @@ pangolin-admin create-catalog prod-catalog-azure --warehouse production-azure ## Related Documentation -- [Storage & Connectivity Architecture](../architecture/storage_and_connectivity.md) - Detailed multi-cloud architecture +- [Storage & Connectivity Architecture](docs/architecture/storage_and_connectivity.md) - Detailed multi-cloud architecture - [PyIceberg Integration Guide](../../planning/pyiceberg_testing_guide.md) - Testing warehouses with PyIceberg - [Authentication Guide](../architecture/authentication.md) - CLI authentication methods @@ -14255,45 +14266,45 @@ This directory contains detailed documentation for the core features of the Pang ## 🗂️ Asset & Data Management Basic data operations and lifecycle management. -- **[Entities](./entities.md)**: Understanding the Pangolin core models. -- **[Asset Management](./asset_management.md)**: Handling Tables, Views, and other assets. -- **[Generic Assets](./generic_assets.md)**: Cataloging ML Models, Videos, and Files. -- **[Modern Table Formats](./table_formats.md)**: Support for Delta Lake, Hudi, and Paimon. -- **[Warehouse Management](./warehouse_management.md)**: Configuring storage backends. -- **[Time Travel](./time_travel.md)**: Querying historical data states. -- **[Tag Management](./tag_management.md)**: Versioning with tags. +- **[Entities](docs/features/entities.md)**: Understanding the Pangolin core models. +- **[Asset Management](docs/features/asset_management.md)**: Handling Tables, Views, and other assets. +- **[Generic Assets](docs/features/generic_assets.md)**: Cataloging ML Models, Videos, and Files. +- **[Modern Table Formats](docs/features/table_formats.md)**: Support for Delta Lake, Hudi, and Paimon. +- **[Warehouse Management](docs/features/warehouse_management.md)**: Configuring storage backends. +- **[Time Travel](docs/features/time_travel.md)**: Querying historical data states. +- **[Tag Management](docs/features/tag_management.md)**: Versioning with tags. ## 🌿 Versioning & Branches Git-like workflows for your data lake. -- **[Branch Management](./branch_management.md)**: Creating and managing branches. -- **[Merge Operations](./merge_operations.md)**: Workflow for 3-way merges. -- **[Merge Conflicts](./merge_conflicts.md)**: Principles of data conflict resolution. +- **[Branch Management](docs/features/branch_management.md)**: Creating and managing branches. +- **[Merge Operations](docs/features/merge_operations.md)**: Workflow for 3-way merges. +- **[Merge Conflicts](docs/features/merge_conflicts.md)**: Principles of data conflict resolution. ## 🛡️ Security & Access Governance and authentication mechanisms. -- **[Multi-Tenancy](./multi_tenancy.md)**: Isolation principles and tenant management. -- **[RBAC](./rbac.md)**: Role-Based Access Control system. -- **[Credential Vending](./iam_roles.md)**: Cloud IAM and STS integration. -- **[Security Concepts](./security_vending.md)**: Concept guide for vending and signing. +- **[Multi-Tenancy](docs/features/multi_tenancy.md)**: Isolation principles and tenant management. +- **[RBAC](docs/features/rbac.md)**: Role-Based Access Control system. +- **[Credential Vending](docs/features/iam_roles.md)**: Cloud IAM and STS integration. +- **[Security Concepts](docs/features/security_vending.md)**: Concept guide for vending and signing. ## 🔍 Discovery & Audit Finding data and tracking changes. -- **[Business Catalog](./business_catalog.md)**: Data discovery portal and business metadata. -- **[Audit Logging](./audit_logs.md)**: Comprehensive action tracking and compliance. +- **[Business Catalog](docs/features/business_catalog.md)**: Data discovery portal and business metadata. +- **[Audit Logging](docs/features/audit_logs.md)**: Comprehensive action tracking and compliance. ## 🌐 Integration & Federation Connecting to external systems. -- **[Federated Catalogs](./federated_catalogs.md)**: Connecting to remote Iceberg catalogs. -- **[Catalog Management](./catalog_management.md)**: Managing local and federated catalogs. +- **[Federated Catalogs](docs/features/federated_catalogs.md)**: Connecting to remote Iceberg catalogs. +- **[Catalog Management](docs/features/catalog_management.md)**: Managing local and federated catalogs. - **[Service Users](./service_users.md)**: API keys for programmatic access. ## 🧪 Integration & Testing Using Pangolin with other tools. -- **[PyIceberg Integration](./pyiceberg_testing.md)**: Guide for Python users. +- **[PyIceberg Integration](docs/features/pyiceberg_testing.md)**: Guide for Python users. ## 🛠️ Maintenance Optimizing and maintaining your data lake. -- **[Maintenance Operations](./maintenance.md)**: Snapshot management and orphan file cleanup. +- **[Maintenance Operations](docs/features/maintenance.md)**: Snapshot management and orphan file cleanup. --- @@ -14924,8 +14935,8 @@ You can interactively register and manage generic assets directly from the **Dat ## 📚 Related Documentation -- **[Modern Table Formats](./table_formats.md)**: Specific guide for cataloging **Delta Lake**, **Hudi**, and **Paimon**. -- **[Asset Management](./asset_management.md)**: General governance guide. +- **[Modern Table Formats](docs/features/table_formats.md)**: Specific guide for cataloging **Delta Lake**, **Hudi**, and **Paimon**. +- **[Asset Management](docs/features/asset_management.md)**: General governance guide. --- @@ -15030,9 +15041,9 @@ pangolin-admin create-warehouse --name "prod-s3" --type "s3" 3. **Use External IDs**: When configuring cross-account `AwsSts` roles, always use an `external_id` to prevent the "confused deputy" problem. ## Related Documentation -- [Warehouse Management](warehouse_management.md) -- [Security & Credential Vending](security_vending.md) -- [Architecture: Signer Trait](../architecture/signer-trait.md) +- [Warehouse Management](docs/features/warehouse_management.md) +- [Security & Credential Vending](docs/features/security_vending.md) +- [Architecture: Signer Trait](docs/architecture/signer-trait.md) --- @@ -15121,7 +15132,7 @@ To run maintenance operations, the user must have the following permissions: In a Data Lakehouse, merging branches is more complex than merging code because the underlying data is massive and mutable. Pangolin implements specific strategies to handle these updates safely. > [!NOTE] -> For the step-by-step API guide and technical details on the 3-Way Merge algorithm, see the **[Merge Operations](./merge_operations.md)** guide. +> For the step-by-step API guide and technical details on the 3-Way Merge algorithm, see the **[Merge Operations](docs/features/merge_operations.md)** guide. ## Types of Changes @@ -15348,7 +15359,7 @@ When using the API or CLI, the tenant context is derived from: - **API Key Context**: For service users. - **X-Pangolin-Tenant Header**: Required for certain administrative operations or when using PyIceberg with cross-tenant access. -For more details on managing tenants, see the **[CLI: Admin Tenants](../cli/admin-tenants.md)** guide. +For more details on managing tenants, see the **[CLI: Admin Tenants](docs/cli/admin-tenants.md)** guide. --- @@ -15776,9 +15787,9 @@ After successful testing: ## Additional Resources -- [Client Configuration](../getting-started/client_configuration.md) - Detailed client setup -- [Getting Started](../getting-started/getting_started.md) - Quick start guide -- [Warehouse Management](./warehouse_management.md) - Warehouse and catalog setup +- [Client Configuration](docs/getting-started/client_configuration.md) - Detailed client setup +- [Getting Started](docs/getting-started/getting_started.md) - Quick start guide +- [Warehouse Management](docs/features/warehouse_management.md) - Warehouse and catalog setup --- @@ -16107,7 +16118,7 @@ POST /api/v1/permissions - [Service Users](../service_users.md) - API key authentication - [Authentication](../authentication.md) - User authentication -- [Audit Logs](./audit_logs.md) - Access monitoring +- [Audit Logs](docs/features/audit_logs.md) - Access monitoring - [Permissions System](../permissions.md) - Permission details @@ -16592,8 +16603,8 @@ AWS_STS_REGIONAL_ENDPOINTS=regional - [Warehouse Management](../warehouse/README.md) - Creating and configuring warehouses - [Authentication](../architecture/authentication.md) - User authentication and tokens -- [Client Configuration](../getting-started/client_configuration.md) - PyIceberg, Spark, Trino setup -- [AWS S3 Storage](../warehouse/s3.md) - S3 storage backend configuration +- [Client Configuration](docs/getting-started/client_configuration.md) - PyIceberg, Spark, Trino setup +- [AWS S3 Storage](docs/warehouse/s3.md) - S3 storage backend configuration --- @@ -17154,9 +17165,9 @@ if len(snapshots) > 1: ## Related Documentation -- [Branch Management](./branch_management.md) - Git-like branching for catalogs -- [Merge Conflicts](../merge_conflicts.md) - Merging branches -- [PyIceberg Testing](./pyiceberg_testing.md) - PyIceberg integration +- [Branch Management](docs/features/branch_management.md) - Git-like branching for catalogs +- [Merge Conflicts](docs/features/merge_conflicts.md) - Merging branches +- [PyIceberg Testing](docs/features/pyiceberg_testing.md) - PyIceberg integration --- @@ -17331,9 +17342,9 @@ curl -X POST http://localhost:8080/api/v1/catalogs \ ## Related Documentation -- [Security & Credential Vending](./security_vending.md) - Detailed credential vending guide -- [AWS S3 Storage](../warehouse/s3.md) - S3 backend configuration -- [Client Configuration](../getting-started/client_configuration.md) - PyIceberg, Spark, Trino setup +- [Security & Credential Vending](docs/features/security_vending.md) - Detailed credential vending guide +- [AWS S3 Storage](docs/warehouse/s3.md) - S3 backend configuration +- [Client Configuration](docs/getting-started/client_configuration.md) - PyIceberg, Spark, Trino setup --- @@ -17346,30 +17357,30 @@ Welcome to Pangolin! This directory contains everything you need to set up, conf ## 🚀 Onboarding Get up and running in minutes. -- **[Quick Start Guide](./getting_started.md)**: A step-by-step walkthrough of your first tenant, catalog, and table. -- **[Evaluating Pangolin](./evaluating-pangolin.md)**: Using `NO_AUTH` mode for rapid local testing. +- **[Quick Start Guide](docs/getting-started/getting_started.md)**: A step-by-step walkthrough of your first tenant, catalog, and table. +- **[Evaluating Pangolin](docs/getting-started/evaluating-pangolin.md)**: Using `NO_AUTH` mode for rapid local testing. ## ⚙️ Configuration Fine-tune Pangolin for your environment. -- **[Environment Variables](./env_vars.md)**: Comprehensive list of all configuration options. +- **[Environment Variables](docs/getting-started/env_vars.md)**: Comprehensive list of all configuration options. - **[Configuration Overview](./configuration.md)**: Principles of runtime and storage setup. -- **[Nested Namespaces](./nested_namespaces.md)**: Guide to creating and managing hierarchical namespaces. -- **[Storage Backend Logic](./storage-backend-logic.md)**: How backends determine which credentials to use (vending vs client-side). +- **[Nested Namespaces](docs/getting-started/nested_namespaces.md)**: Guide to creating and managing hierarchical namespaces. +- **[Storage Backend Logic](docs/getting-started/storage-backend-logic.md)**: How backends determine which credentials to use (vending vs client-side). - **[Dependencies](./dependencies.md)**: System requirements and library overview. ## 🔌 Client Integration Connect your favorite tools to the Pangolin REST Catalog. -- **[Client Configuration](./client_configuration.md)**: Setup guides for PyIceberg, PySpark, and Trino. +- **[Client Configuration](docs/getting-started/client_configuration.md)**: Setup guides for PyIceberg, PySpark, and Trino. ## 🔐 Authentication Modes Choose your security model. -- **[Auth Mode](./auth-mode.md)**: Standard operational mode with user authentication. -- **[No Auth Mode](./no-auth-mode.md)**: For local development and testing. +- **[Auth Mode](docs/getting-started/auth-mode.md)**: Standard operational mode with user authentication. +- **[No Auth Mode](docs/getting-started/no-auth-mode.md)**: For local development and testing. ## 🚢 Deployment Move from local testing to production. - **[Deployment Guide](./deployment.md)**: Instructions for local, Docker, and production environments. -- **[Docker Deployment](./docker_deployment.md)**: Detailed Docker setup and configuration. +- **[Docker Deployment](docs/getting-started/docker_deployment.md)**: Detailed Docker setup and configuration. --- @@ -17383,7 +17394,7 @@ cd pangolin docker-compose up -d ``` -Visit the [Quick Start Guide](./getting_started.md) to perform your first data operations! +Visit the [Quick Start Guide](docs/getting-started/getting_started.md) to perform your first data operations! --- @@ -17894,7 +17905,7 @@ Pangolin follow a "Configuration-over-Code" philosophy, allowing for flexible de Most settings are managed via **Environment Variables** at startup. This includes everything from the port number to the metadata persistence backend. -- For a complete list of variables, see **[Environment Variables](../environment-variables.md)**. +- For a complete list of variables, see **[Environment Variables](docs/environment-variables.md)**. - For deployment-specific patterns, see **[Deployment Guide](./deployment.md)**. ## Client Configuration Discovery @@ -17929,7 +17940,7 @@ GET http://localhost:8080/v1/config?warehouse=main While core system configuration uses environment variables, **Storage Connectors** (Warehouses) are configured dynamically via the Admin API/CLI. This allows you to add or modify storage locations without restarting the Pangolin service. -- To learn how to configure storage, see **[Warehouse Management](../features/warehouse_management.md)**. +- To learn how to configure storage, see **[Warehouse Management](docs/features/warehouse_management.md)**. --- @@ -18056,7 +18067,7 @@ If using Docker deployment: ## Related Documentation -- [Getting Started](./getting_started.md) - Quick start guide +- [Getting Started](docs/getting-started/getting_started.md) - Quick start guide - [Configuration](./configuration.md) - Configuration options - [Deployment](./deployment.md) - Production deployment @@ -18143,15 +18154,15 @@ export DATABASE_URL="postgresql://user:pass@db-host:5432/pangolin" Ensure `PANGOLIN_NO_AUTH` is NOT set to `true`. Set a strong `PANGOLIN_JWT_SECRET`. ### 3. Use Cloud IAM Roles -For storage access, prefer `AwsSts` or `AzureSas` vending strategies over static keys. See [Credential Vending](../features/iam_roles.md). +For storage access, prefer `AwsSts` or `AzureSas` vending strategies over static keys. See [Credential Vending](docs/features/iam_roles.md). ### 4. Monitoring & Logging Set `RUST_LOG=info` or `debug` and integrate the container logs with your logging provider. ## Related Documentation -- [Environment Variables](./env_vars.md) -- [Client Configuration](./client_configuration.md) -- [Multi-Tenancy](../features/multi_tenancy.md) +- [Environment Variables](docs/getting-started/env_vars.md) +- [Client Configuration](docs/getting-started/client_configuration.md) +- [Multi-Tenancy](docs/features/multi_tenancy.md) --- @@ -18444,7 +18455,7 @@ curl -X POST http://localhost:8080/api/v1/warehouses \ } }' ``` -*Note: We use `vending_strategy: AwsStatic` for this quick start. For production with IAM Role assumption, see [Credential Vending](../features/iam_roles.md).* +*Note: We use `vending_strategy: AwsStatic` for this quick start. For production with IAM Role assumption, see [Credential Vending](docs/features/iam_roles.md).* ### 3. Create a Catalog A catalog references a warehouse and specifies a storage location. The catalog name is what clients use in their connection URIs. @@ -18552,9 +18563,9 @@ Now, `main` will contain the schema updates made in `dev`. ## Next Steps -- Explore [Branch Management](../features/branch_management.md) for advanced strategies. -- Learn about [Credential Vending](../features/iam_roles.md). -- Set up [Client Configuration](client_configuration.md) for PyIceberg, PySpark, Trino, or Dremio. +- Explore [Branch Management](docs/features/branch_management.md) for advanced strategies. +- Learn about [Credential Vending](docs/features/iam_roles.md). +- Set up [Client Configuration](docs/getting-started/client_configuration.md) for PyIceberg, PySpark, Trino, or Dremio. ## Production Setup @@ -18625,9 +18636,9 @@ Response (403 Forbidden): ## Next Steps -- [PyIceberg Testing Guide](../features/pyiceberg_testing.md) - Comprehensive PyIceberg testing -- [Client Configuration](./client_configuration.md) - Configure various Iceberg clients -- [Warehouse Management](../features/warehouse_management.md) - Manage warehouses and catalogs +- [PyIceberg Testing Guide](docs/features/pyiceberg_testing.md) - Comprehensive PyIceberg testing +- [Client Configuration](docs/getting-started/client_configuration.md) - Configure various Iceberg clients +- [Warehouse Management](docs/features/warehouse_management.md) - Manage warehouses and catalogs - [API Reference](../api/) - Complete API documentation @@ -19202,7 +19213,7 @@ This section documents verified issues, limitations, and architectural quirks pr ## v0.4.0 -* [SQL Backend Token Listing (SQLite/Postgres)](./token_listing_sqlite_join.md) +* [SQL Backend Token Listing (SQLite/Postgres)](docs/known-issues/token_listing_sqlite_join.md) * **Description**: Active token lists may be empty for Root users or ephemeral accounts when using SQL backends due to a strict `JOIN` dependency. * **Status**: Identified. Fix proposed for v0.5.x. @@ -19356,23 +19367,23 @@ Choose the guide that matches your environment: ### Authenticated (Production) For multi-tenant environments where security is enforced via JWT tokens. -- **[Credential Vending](./auth_vended_creds.md)** (Recommended): Pangolin manages storage keys. -- **[Client-Provided Credentials](./auth_client_creds.md)**: You provide storage keys to the client. +- **[Credential Vending](docs/pyiceberg/auth_vended_creds.md)** (Recommended): Pangolin manages storage keys. +- **[Client-Provided Credentials](docs/pyiceberg/auth_client_creds.md)**: You provide storage keys to the client. ### Service Users (Machine Identity) For automation and CI/CD where persistent access is needed. -- **[API Key Authentication](./auth_api_key.md)**: Using Service User API keys. +- **[API Key Authentication](docs/pyiceberg/auth_api_key.md)**: Using Service User API keys. ### No-Auth (Evaluation) For rapid local testing and development. -- **[Credential Vending](./no_auth_vended_creds.md)**: Test the vending flow without RBAC. -- **[Client-Provided Credentials](./no_auth_client_creds.md)**: Basic local storage connectivity. +- **[Credential Vending](docs/pyiceberg/no_auth_vended_creds.md)**: Test the vending flow without RBAC. +- **[Client-Provided Credentials](docs/pyiceberg/no_auth_client_creds.md)**: Basic local storage connectivity. --- ## ☁️ Multi-Cloud Support Detailed configuration for different storage backends. -- **[Azure & GCP Integration](./multi_cloud.md)**: Connecting to ADLS Gen2 and Google Cloud Storage. +- **[Azure & GCP Integration](docs/pyiceberg/multi_cloud.md)**: Connecting to ADLS Gen2 and Google Cloud Storage. --- @@ -19747,21 +19758,21 @@ catalog = load_catalog( This section contains detailed reference guides for all major Pangolin concepts and operations, covering API, CLI, Python SDK, and UI usage. ## Core Concepts -* [**Tenants**](./tenants.md): Managing isolation units and context switching. -* [**Users**](./users.md): Identity management for humans. -* [**Access Control**](./access_control.md): RBAC (Roles) and ABAC/TBAC (Tags). +* [**Tenants**](docs/reference/tenants.md): Managing isolation units and context switching. +* [**Users**](docs/reference/users.md): Identity management for humans. +* [**Access Control**](docs/reference/access_control.md): RBAC (Roles) and ABAC/TBAC (Tags). * [**Service Users**](./service_users.md): Programmatic access keys for automation. -* [**Tokens**](./tokens.md): Session management and authentication. -* [**Auditing**](./auditing.md): Governance logs and monitoring. +* [**Tokens**](docs/reference/tokens.md): Session management and authentication. +* [**Auditing**](docs/reference/auditing.md): Governance logs and monitoring. ## Storage & Catalogs -* [**Warehouses**](./warehouses.md): Configuring object storage (S3, Azure, GCS). -* [**Catalogs**](./catalogs.md): Managing local and federated Iceberg catalogs. +* [**Warehouses**](docs/reference/warehouses.md): Configuring object storage (S3, Azure, GCS). +* [**Catalogs**](docs/reference/catalogs.md): Managing local and federated Iceberg catalogs. ## Data Management -* [**Assets**](./assets.md): Managing tables, views, and generic files. +* [**Assets**](docs/reference/assets.md): Managing tables, views, and generic files. * [**Business Metadata**](./metadata.md): Tagging and custom properties. -* [**Version Control**](./version_control.md): Branching, merging, and conflict resolution. +* [**Version Control**](docs/reference/version_control.md): Branching, merging, and conflict resolution. --- @@ -20944,20 +20955,20 @@ Welcome to the documentation for the Pangolin Management UI. This web interface ### [General Overview](./overview.md) Start here to understand the interface layout, theme switching, and the core dashboard experience. -### [Data Management](./data_management.md) +### [Data Management](docs/ui/data_management.md) Learn how to browse your data, manage branches (Git-for-Data), and handle merge operations. - Data Explorer - Branching & Tagging - Merge Operations & Conflict Resolution - Table Maintenance -### [Discovery & Governance](./discovery_governance.md) +### [Discovery & Governance](docs/ui/discovery_governance.md) Tools for finding data and auditing system activity. - Data Discovery Portal - Access Request Workflow - Audit Log Viewer -### [Administration](./administration.md) +### [Administration](docs/ui/administration.md) Configure the foundational resources of your Pangolin instance. - Multi-Tenant Management - Warehouse & Catalog Configuration @@ -21273,7 +21284,7 @@ For local testing without an identity provider. Username and Bcrypt-hashed password management for standard deployments. ### 3. OAuth -One-click login with Google, GitHub, or Microsoft (configured in [System Settings](./administration.md)). +One-click login with Google, GitHub, or Microsoft (configured in [System Settings](docs/ui/administration.md)). --- @@ -21286,7 +21297,7 @@ This directory contains utility documentation for maintaining and working with t ## Available Guides -### [Regenerating OpenAPI Documentation](./regenerating-openapi.md) +### [Regenerating OpenAPI Documentation](docs/utilities/regenerating-openapi.md) Complete guide for regenerating the OpenAPI specification files (JSON and YAML) after making changes to API handlers or models. **Quick Commands**: @@ -21838,10 +21849,10 @@ The `use_sts` boolean field is deprecated. Use `vending_strategy` instead. | Storage | Status | Best For | |---------|--------|----------| -| [AWS S3](s3.md) | ✅ Production | Most common, excellent performance | -| [Azure Blob](azure.md) | ✅ Production | Azure-native deployments | -| [Google Cloud Storage](gcs.md) | ✅ Production | GCP-native deployments | -| [Local Filesystem](local.md) | ⚠️ Dev/Test | Local development & testing | +| [AWS S3](docs/warehouse/s3.md) | ✅ Production | Most common, excellent performance | +| [Azure Blob](docs/warehouse/azure.md) | ✅ Production | Azure-native deployments | +| [Google Cloud Storage](docs/warehouse/gcs.md) | ✅ Production | GCP-native deployments | +| [Local Filesystem](docs/warehouse/local.md) | ⚠️ Dev/Test | Local development & testing | ## Quick Start @@ -21916,9 +21927,9 @@ When a catalog has no warehouse, clients must configure storage themselves. **PySpark**: Configure Hadoop filesystem properties See individual storage guides for details: -- [S3 Client Configuration](s3.md) -- [Azure Client Configuration](azure.md) -- [GCS Client Configuration](gcs.md) +- [S3 Client Configuration](docs/warehouse/s3.md) +- [Azure Client Configuration](docs/warehouse/azure.md) +- [GCS Client Configuration](docs/warehouse/gcs.md) ## Best Practices @@ -21976,9 +21987,9 @@ Error: No credentials provided ## Next Steps -- [S3 Warehouse Configuration](s3.md) -- [Azure Blob Warehouse Configuration](azure.md) -- [GCS Warehouse Configuration](gcs.md) +- [S3 Warehouse Configuration](docs/warehouse/s3.md) +- [Azure Blob Warehouse Configuration](docs/warehouse/azure.md) +- [GCS Warehouse Configuration](docs/warehouse/gcs.md) - [Backend Storage Options](../backend_storage/README.md) @@ -22216,8 +22227,8 @@ Error: This request is not authorized to perform this operation ## Next Steps -- [S3 Warehouse](s3.md) -- [GCS Warehouse](gcs.md) +- [S3 Warehouse](docs/warehouse/s3.md) +- [GCS Warehouse](docs/warehouse/gcs.md) - [Warehouse Concept](README.md) @@ -22481,8 +22492,8 @@ Error: 403 Forbidden ## Next Steps -- [S3 Warehouse](s3.md) -- [Azure Warehouse](azure.md) +- [S3 Warehouse](docs/warehouse/s3.md) +- [Azure Warehouse](docs/warehouse/azure.md) - [Warehouse Concept](README.md) @@ -22897,7 +22908,7 @@ aws cloudtrail create-trail \ ## Next Steps -- [Azure Blob Warehouse](azure.md) -- [GCS Warehouse](gcs.md) +- [Azure Blob Warehouse](docs/warehouse/azure.md) +- [GCS Warehouse](docs/warehouse/gcs.md) - [Warehouse Concept](README.md) - [Backend Storage](../backend_storage/README.md) diff --git a/pangolin_ui/Dockerfile b/pangolin_ui/Dockerfile index 23ca725..08ee1eb 100644 --- a/pangolin_ui/Dockerfile +++ b/pangolin_ui/Dockerfile @@ -1,5 +1,5 @@ # Build Stage -FROM node:20-alpine as builder +FROM node:20-alpine AS builder WORKDIR /usr/src/pangolin-ui @@ -15,15 +15,33 @@ COPY . . # Build the SvelteKit app RUN npm run build +# Drop the build toolchain from node_modules. +# +# The runtime stage used to copy the *whole* node_modules across, so the +# published image shipped all 22 devDependencies - vite, playwright, +# svelte-check, vitest, the tailwind toolchain - none of which run in +# production. adapter-node needs only the `dependencies` entries alongside its +# bundled output, and this project has exactly one. +RUN npm prune --omit=dev + # Runtime Stage FROM node:20-alpine +LABEL org.opencontainers.image.title="Pangolin UI" \ + org.opencontainers.image.description="Web interface for the Pangolin Iceberg catalog" \ + org.opencontainers.image.source="https://github.com/AlexMercedCoder/pangolin" \ + org.opencontainers.image.licenses="MIT" + WORKDIR /app # Copy built files -COPY --from=builder /usr/src/pangolin-ui/build ./build -COPY --from=builder /usr/src/pangolin-ui/package.json ./package.json -COPY --from=builder /usr/src/pangolin-ui/node_modules ./node_modules +COPY --from=builder --chown=node:node /usr/src/pangolin-ui/build ./build +COPY --from=builder --chown=node:node /usr/src/pangolin-ui/package.json ./package.json +COPY --from=builder --chown=node:node /usr/src/pangolin-ui/node_modules ./node_modules + +# The node image ships an unprivileged `node` user; the image had no USER +# directive at all, so the server ran as root. +USER node # Expose port (default 3000) EXPOSE 3000 diff --git a/pypangolin_docs.md b/pypangolin_docs.md index 528cfde..430425f 100644 --- a/pypangolin_docs.md +++ b/pypangolin_docs.md @@ -1,3 +1,14 @@ +` marker naming + the file it came from. + + Relative links inside are relative to the *original* file's location and do + not resolve here. Edit the source file, not this one. +--> + # Admin & System @@ -218,13 +229,13 @@ PyPangolin provides secure credential management for database connections using ## Main Guide -- **[Database Connections Overview](../connections.md)** - Complete guide to database connection management +- **[Database Connections Overview](pypangolin/docs/connections.md)** - Complete guide to database connection management ## SQL Databases - **[PostgreSQL](postgresql.md)** ✅ *Tested* - Open-source relational database -- **[MySQL](mysql.md)** ✅ *Tested* - Popular relational database -- **[Amazon Redshift](redshift.md)** ⚠️ *Untested* - Cloud data warehouse (Postgres-compatible) +- **[MySQL](pypangolin/docs/connections/mysql.md)** ✅ *Tested* - Popular relational database +- **[Amazon Redshift](pypangolin/docs/connections/redshift.md)** ⚠️ *Untested* - Cloud data warehouse (Postgres-compatible) ## NoSQL Databases @@ -232,13 +243,13 @@ PyPangolin provides secure credential management for database connections using ## Cloud Data Warehouses -- **[Snowflake](snowflake.md)** ⚠️ *Untested* - Cloud data platform -- **[Azure Synapse](synapse.md)** ⚠️ *Untested* - Microsoft analytics service -- **[Google BigQuery](bigquery.md)** ⚠️ *Untested* - Serverless data warehouse +- **[Snowflake](pypangolin/docs/connections/snowflake.md)** ⚠️ *Untested* - Cloud data platform +- **[Azure Synapse](pypangolin/docs/connections/synapse.md)** ⚠️ *Untested* - Microsoft analytics service +- **[Google BigQuery](pypangolin/docs/connections/bigquery.md)** ⚠️ *Untested* - Serverless data warehouse ## Analytics Platforms -- **[Dremio](dremio.md)** ✅ *Tested* - Data lakehouse platform with Arrow Flight +- **[Dremio](pypangolin/docs/connections/dremio.md)** ✅ *Tested* - Data lakehouse platform with Arrow Flight ## Quick Start @@ -1439,19 +1450,19 @@ PyPangolin provides secure credential management for database connections using ### SQL Databases - **[PostgreSQL](connections/postgresql.md)** - Open-source relational database ✅ *Tested* -- **[MySQL](connections/mysql.md)** - Popular relational database ✅ *Tested* -- **[Amazon Redshift](connections/redshift.md)** - Cloud data warehouse (Postgres-compatible) ⚠️ *Untested* +- **[MySQL](pypangolin/docs/connections/mysql.md)** - Popular relational database ✅ *Tested* +- **[Amazon Redshift](pypangolin/docs/connections/redshift.md)** - Cloud data warehouse (Postgres-compatible) ⚠️ *Untested* ### NoSQL Databases - **[MongoDB](connections/mongodb.md)** - Document database ✅ *Tested* ### Cloud Data Warehouses -- **[Snowflake](connections/snowflake.md)** - Cloud data platform ⚠️ *Untested* -- **[Azure Synapse](connections/synapse.md)** - Microsoft analytics service ⚠️ *Untested* -- **[Google BigQuery](connections/bigquery.md)** - Serverless data warehouse ⚠️ *Untested* +- **[Snowflake](pypangolin/docs/connections/snowflake.md)** - Cloud data platform ⚠️ *Untested* +- **[Azure Synapse](pypangolin/docs/connections/synapse.md)** - Microsoft analytics service ⚠️ *Untested* +- **[Google BigQuery](pypangolin/docs/connections/bigquery.md)** - Serverless data warehouse ⚠️ *Untested* ### Analytics Platforms -- **[Dremio](connections/dremio.md)** - Data lakehouse platform with Arrow Flight ✅ *Tested* +- **[Dremio](pypangolin/docs/connections/dremio.md)** - Data lakehouse platform with Arrow Flight ✅ *Tested* ## Quick Start @@ -1600,9 +1611,9 @@ encryption_key = secret['data']['data']['encryption_key'] ## See Also - [PostgreSQL Guide](connections/postgresql.md) -- [MySQL Guide](connections/mysql.md) +- [MySQL Guide](pypangolin/docs/connections/mysql.md) - [MongoDB Guide](connections/mongodb.md) -- [Dremio Guide](connections/dremio.md) +- [Dremio Guide](pypangolin/docs/connections/dremio.md) --- diff --git a/scripts/build_docker_sequential.sh b/scripts/build_docker_sequential.sh index 5ad5178..3b93329 100755 --- a/scripts/build_docker_sequential.sh +++ b/scripts/build_docker_sequential.sh @@ -41,22 +41,40 @@ echo # existing release tag replaces what users pulled yesterday with something else # under the same name, which is the one thing a version number is supposed to # prevent. Set ALLOW_OVERWRITE=1 to proceed anyway. +# An image already published at $VERSION is skipped rather than rebuilt, so a +# run that dies partway through can simply be run again. +# +# This was an all-or-nothing abort: if *any* of the three tags existed the +# script refused to start. Publishing 0.8.0 hit exactly that - the API image +# pushed, the CLI image failed to compile (Dockerfile.tools was still on +# rust:1.88), and the only way to finish the release was ALLOW_OVERWRITE=1, +# which would also have re-pushed the good API image over itself. A partial +# failure must be resumable without arming the one flag that lets you clobber a +# published artefact. +ALREADY_PUBLISHED="" if [[ -z "$DRY_RUN" && "${ALLOW_OVERWRITE:-}" != "1" ]]; then for repo in pangolin-api pangolin-cli pangolin-ui; do code=$(curl -s -o /dev/null -w "%{http_code}" \ "https://hub.docker.com/v2/repositories/alexmerced/$repo/tags/$VERSION" || echo 000) if [[ "$code" == "200" ]]; then - echo "error: alexmerced/$repo:$VERSION is already published." >&2 - echo " Bump the version, or set ALLOW_OVERWRITE=1 if you are certain." >&2 - exit 1 + echo " alexmerced/$repo:$VERSION is already published - skipping." + ALREADY_PUBLISHED="$ALREADY_PUBLISHED $repo" fi done - echo "None of the three tags exist yet; proceeding." + if [[ -n "$ALREADY_PUBLISHED" ]]; then + echo " (set ALLOW_OVERWRITE=1 to rebuild and re-push them.)" + else + echo "None of the three tags exist yet; proceeding." + fi echo fi build() { local name="$1" dockerfile="$2" context="$3" step="$4" + if [[ " $ALREADY_PUBLISHED " == *" $name "* ]]; then + echo "--- Skipping ${name} (${step}/3): already at ${VERSION} ---" + return 0 + fi echo "--- Building ${name} (${step}/3) ---" $DRY_RUN docker buildx build \ --platform linux/amd64,linux/arm64 \ @@ -72,4 +90,4 @@ build pangolin-cli pangolin/Dockerfile.tools pangolin 2 build pangolin-ui pangolin_ui/Dockerfile pangolin_ui 3 echo -echo "All three images published at ${VERSION} and latest." +echo "All three images are now published at ${VERSION} and latest." diff --git a/scripts/check_doc_links.sh b/scripts/check_doc_links.sh new file mode 100755 index 0000000..6447963 --- /dev/null +++ b/scripts/check_doc_links.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Every relative Markdown link must resolve. +# +# The documentation carried 274 broken links: files moved between `docs/` +# subdirectories and the links that pointed at them were never updated, plus a +# set of references into a `planning/` directory that is not in this repository +# at all. A link that 404s is worse than no link - it sends a reader looking for +# something that does not exist, and it makes the surrounding text look +# maintained when it is not. +# +# Excluded, deliberately: +# * pangolin_docs.md / pypangolin_docs.md - generated concatenations whose +# inner links are relative to the original files' locations and cannot +# resolve in the flattened document. They carry a header saying so. +# * pangolin/target/ - build artefacts, including vendored copies of README +# files from packaged crates. +# * node_modules/ - not ours. +# +# Usage: scripts/check_doc_links.sh (run from the repository root) + +set -uo pipefail + +broken=0 + +while IFS= read -r file; do + dir=$(dirname "$file") + # Markdown links to a .md target, relative only. Anchors and URLs are out of + # scope: an anchor needs heading parsing and a URL needs the network, and + # both are a different check from "does this file exist". + grep -oE '\]\((\.\./|\./)?[a-zA-Z0-9_./-]+\.md\)' "$file" 2>/dev/null \ + | sed 's/](//; s/)$//' \ + | while IFS= read -r link; do + if [[ ! -f "$dir/$link" ]]; then + echo "::error file=$file::broken link -> $link" + echo " $file -> $link" + fi + done +done < <( + find . -name '*.md' \ + -not -path './node_modules/*' \ + -not -path '*/node_modules/*' \ + -not -path './pangolin/target/*' \ + -not -name 'pangolin_docs.md' \ + -not -name 'pypangolin_docs.md' +) > /tmp/pangolin_doc_links.$$ 2>&1 + +if [[ -s /tmp/pangolin_doc_links.$$ ]]; then + cat /tmp/pangolin_doc_links.$$ + broken=$(grep -c -- '->' /tmp/pangolin_doc_links.$$ || true) + rm -f /tmp/pangolin_doc_links.$$ + echo + echo "error: $((broken / 2)) broken documentation link(s)." >&2 + echo " Either point them at the file that exists, or unlink the text" >&2 + echo " if the target is not in this repository." >&2 + exit 1 +fi + +rm -f /tmp/pangolin_doc_links.$$ +echo "every relative documentation link resolves"