Update pre-commit hooks, change lockfile to use uv pip compile (#278)

* Update pre-commit hooks, change lockfile to use uv pip compile

uv is much faster and thus more appropriate for a mandatory commit hook.

NB: setuptools is now in the lockfile, because uv does not have a
"unsafe package exclusion" facility.

duckdb was downgraded to v0.10.0, which is the last version without
segfaults on cleanup with registered file systems.

* Update lockfile generation script to use `uv pip compile`

In place of regular pip-compile.

* Access error message safely, rework test assumptions

We were getting some bad accesses (key errors) on empty-body API responses
from the lakeFS server, causing a crash in the error handling.

* Silence git-revision plugin warnings, add date fallback

This solution lets us upgrade the mkdocs tech stack, while not abandoning
strict mode just for the git-revision plugin.

The root cause for the previous failures is not yet determined, but has
something to do with recent mkdocs/mkdocs-material updates.

* Bump lakefs-sdk to latest

* fix(ci): Allow slight decreases in coverage for PRs

---------

Co-authored-by: Adrian Rumpold <a.rumpold@gmail.com>

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-json
@@ -12,14 +12,14 @@ repos:
       - id: end-of-file-fixer
       - id: mixed-line-ending
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       # See https://github.com/pre-commit/mirrors-mypy/blob/main/.pre-commit-hooks.yaml
       - id: mypy
         types_or: [python, pyi]
         args: [--ignore-missing-imports, --scripts-are-modules]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.4.3
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -34,13 +34,14 @@ repos:
     rev: 0.4.1
     hooks:
       - id: pydoclint
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.1
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.1.39
     hooks:
       - id: pip-compile
         name: pip-compile requirements-dev.txt
         args:
           - --no-annotate
+          - --no-strip-extras
           - --extra=dev
           - --output-file=requirements-dev.txt
           - pyproject.toml
@@ -49,6 +50,7 @@ repos:
         name: pip-compile requirements-docs.txt
         args:
           - --no-annotate
+          - --no-strip-extras
           - --extra=docs
           - --output-file=requirements-docs.txt
           - pyproject.toml

codecov.yml

@@ -0,0 +1,6 @@
+coverage:
+  status:
+    project:
+      default:
+        # Allow slight decreases in coverage for PRs
+        threshold: "1%"

docs/quickstart.md

@@ -2,10 +2,10 @@
 
 Welcome! This quickstart guide will get you up and running with lakeFS-spec by showing you how to
 
-1. [install the `lakefs-spec` package](#installing),
+1. [install the `lakefs-spec` package](#installing-lakefs-spec),
 1. [spin up a local lakeFS server](#spinning-up-a-local-lakefs-instance),
 1. [create a lakeFS repository for experimentation](#create-a-lakefs-repository), and
-1. [perform basic file system operations](#using-the-lakefs-fsspec-file-system)
+1. [perform basic file system operations](#using-the-lakefs-file-system)
 in a lakeFS repository using lakeFS-spec.
 
 ??? info "Prerequisites"

hack/lock-deps.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
 INFILE=pyproject.toml
-COMMON_OPTIONS=(--no-annotate)
+COMMON_OPTIONS=(--no-annotate --no-strip-extras)
 
 # Treat positional arguments as packages to be upgraded
 while [[ $# -gt 0 ]]; do
@@ -10,5 +10,5 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Lock (and, if specified, upgrade) packages
-pip-compile "${COMMON_OPTIONS[@]}" --extra=dev --output-file=requirements-dev.txt "$INFILE"
-pip-compile "${COMMON_OPTIONS[@]}" --extra=docs --output-file=requirements-docs.txt "$INFILE"
+uv pip compile "${COMMON_OPTIONS[@]}" --extra=dev --output-file=requirements-dev.txt "$INFILE"
+uv pip compile "${COMMON_OPTIONS[@]}" --extra=docs --output-file=requirements-docs.txt "$INFILE"

mkdocs.yml

@@ -78,6 +78,8 @@ plugins:
   - search:
   - git-revision-date-localized:
       type: iso_date
+      strict: false
+      fallback_to_build_date: true
   - include_dir_to_nav:
       file_pattern: '.*\.(md|ipynb)$'
 

pyproject.toml

@@ -55,7 +55,7 @@ dev = [
     # for integration tests.
     "pandas[parquet]",
     "polars",
-    "duckdb",
+    "duckdb<=0.10.0",  # due to https://github.com/duckdb/duckdb/issues/11875
 ]
 docs = [
     "mkdocs",

requirements-dev.txt

@@ -1,45 +1,41 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile --extra=dev --no-annotate --output-file=requirements-dev.txt pyproject.toml
-#
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --no-annotate --no-strip-extras --extra=dev --output-file=requirements-dev.txt pyproject.toml
 aenum==3.1.15
-build==1.0.3
+annotated-types==0.6.0
+build==1.2.1
 cfgv==3.4.0
 click==8.1.7
-coverage[toml]==7.4.1
+coverage[toml]==7.5.1
 distlib==0.3.8
 docstring-parser-fork==0.0.5
-duckdb==0.9.2
-filelock==3.13.1
-fsspec==2024.2.0
-identify==2.5.33
+duckdb==0.10.0
+filelock==3.14.0
+fsspec==2024.3.1
+identify==2.5.36
 iniconfig==2.0.0
-lakefs==0.3.0
-lakefs-sdk==1.9.1
+lakefs==0.6.0
+lakefs-sdk==1.21.0
 nodeenv==1.8.0
-numpy==1.26.3
-packaging==23.2
-pandas[parquet]==2.2.0
-platformdirs==4.2.0
-pluggy==1.4.0
-polars==0.20.7
-pre-commit==3.6.0
-pyarrow==15.0.0
-pydantic==1.10.14
-pydoclint==0.3.9
-pyproject-hooks==1.0.0
-pytest==8.0.0
-pytest-cov==4.1.0
-python-dateutil==2.8.2
+numpy==1.26.4
+packaging==24.0
+pandas[parquet]==2.2.2
+platformdirs==4.2.1
+pluggy==1.5.0
+polars==0.20.23
+pre-commit==3.7.0
+pyarrow==16.0.0
+pydantic==2.7.1
+pydantic-core==2.18.2
+pydoclint==0.4.1
+pyproject-hooks==1.1.0
+pytest==8.2.0
+pytest-cov==5.0.0
+python-dateutil==2.9.0.post0
 pytz==2024.1
 pyyaml==6.0.1
+setuptools==68.2.2
 six==1.16.0
-typing-extensions==4.9.0
-tzdata==2023.4
+typing-extensions==4.11.0
+tzdata==2024.1
 urllib3==2.0.7
-virtualenv==20.25.0
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+virtualenv==20.26.1