Update dependencies, adapt ruff, release 0.0.4.

Changelog.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.0.4] - 2024-06-15
+
+### Added
+
+- Added support for stratum1 servers to use S3 as a backend. If any server has S3 as a backend, one has to set the `repos` for the scraping to ensure that the repositories in question are scraped. This is due to the fact that the S3 backend does not offer a repository list to clients (or the scraper). Also note that GeoAPI is not supported for S3 backends and will always return `GeoAPIStatus.NO_RESPONSE`.
+
 ## [0.0.3] - 2024-01-21
 
 ### Added

cvmfsscraper/http_get_models.py

@@ -1,4 +1,5 @@
 """Pydantic models for CVMFS HTTP responses."""
+
 import re
 from datetime import datetime
 from enum import Enum

cvmfsscraper/repository.py

@@ -1,10 +1,15 @@
 """A CVMFS repository."""
+
 from typing import TYPE_CHECKING, Dict, cast
 
 import structlog
 
 from cvmfsscraper.exceptions import CVMFSFetchError
-from cvmfsscraper.http_get_models import Endpoints, GetCVMFSPublished, GetCVMFSStatusJSON
+from cvmfsscraper.http_get_models import (
+    Endpoints,
+    GetCVMFSPublished,
+    GetCVMFSStatusJSON,
+)
 
 log = structlog.getLogger(__name__)
 
@@ -74,7 +79,12 @@ def __str__(self) -> str:
 
     def scrape(self) -> None:
         """Scrape the repository."""
-        log.debug("Scraping repository", server=self.server.name, name=self.name, url=self.path)
+        log.debug(
+            "Scraping repository",
+            server=self.server.name,
+            name=self.name,
+            url=self.path,
+        )
         try:
             cvmfspublished = self.fetch_cvmfspublished()
             self.parse_cvmfspublished(cvmfspublished)

cvmfsscraper/tests/base.py

@@ -1,4 +1,5 @@
 """Base testing framework for cvmfsscraper tests."""
+
 import os
 import urllib.error
 import urllib.request

cvmfsscraper/tools.py

@@ -2,6 +2,7 @@
 
 This package is not allowed to import any other packages from cvmfsscraper.
 """
+
 import sys
 import urllib.request
 from typing import Any

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cvmfs-server-scraper"
-version = "0.0.3"
+version = "0.0.4"
 description = "Scrape metadata from CVMFS Stratum servers."
 authors = ["Terje Kvernes <terje@kvernes.no>"]
 maintainers = ["Terje Kvernes <terje@kvernes.no>"]
@@ -11,7 +11,7 @@ repository = "https://github.com/eessi/cvmfs-server-scraper"
 documentation = "https://github.com/eessi/cvmfs-server-scraper"
 keywords = ["cvmfs", "scrape", "eessi"]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Information Technology",
     "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
     "Programming Language :: Python",
@@ -51,6 +51,38 @@ fail_under = 100
 show_missing = true
 
 [tool.ruff]
+line-length = 99
+# Assume Python 3.10.
+target-version = "py310"
+
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    "migrations",
+]
+
+[tool.ruff.lint]
 select = [
     "A", # flake8-builtins
     #    "ANN", # annotations
@@ -70,6 +102,7 @@ select = [
     #    "PLW", # pylint-warning
     "W", # warnings        
 ]
+
 ignore = [
     # D203 (1 blank line required before class docstring) is incompatible with
     # D211 (No blank lines allowed before class docstring). We ignore D203.
@@ -153,42 +186,12 @@ fixable = [
     "YTT",
 ]
 unfixable = []
-
-# Exclude a variety of commonly ignored directories.
-exclude = [
-    ".bzr",
-    ".direnv",
-    ".eggs",
-    ".git",
-    ".git-rewrite",
-    ".hg",
-    ".mypy_cache",
-    ".nox",
-    ".pants.d",
-    ".pytype",
-    ".ruff_cache",
-    ".svn",
-    ".tox",
-    ".venv",
-    "__pypackages__",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
-    "node_modules",
-    "venv",
-    "migrations",
-]
 per-file-ignores = {}
 
-line-length = 99
-
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-# Assume Python 3.10.
-target-version = "py310"
 
-[tool.ruff.mccabe]
+[tool.lint.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
 max-complexity = 10