Skip to content

Commit

Permalink
Remove black and isort, moving to ruff only.
Browse files Browse the repository at this point in the history
  • Loading branch information
terjekv committed Jan 21, 2024
1 parent 7bae80a commit db1172c
Show file tree
Hide file tree
Showing 9 changed files with 54 additions and 47 deletions.
10 changes: 3 additions & 7 deletions cvmfsscraper/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
)


def set_log_level(level: int) -> None:
def set_log_level(level: int) -> None: # pragma: no cover
"""Set the log level for the library.
This function allows the consumer of the library to set the desired log level.
Expand Down Expand Up @@ -76,15 +76,11 @@ def scrape(
with ThreadPoolExecutor(max_workers=10) as executor:
for server in stratum1_servers:
processes.append(
executor.submit(
scrape_server, server, repos, ignore_repos, is_stratum0=False
)
executor.submit(scrape_server, server, repos, ignore_repos, is_stratum0=False)
)
for server in stratum0_servers:
processes.append(
executor.submit(
scrape_server, server, repos, ignore_repos, is_stratum0=True
)
executor.submit(scrape_server, server, repos, ignore_repos, is_stratum0=True)
)

for task in as_completed(processes):
Expand Down
4 changes: 1 addition & 3 deletions cvmfsscraper/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@
class CVMFSScraperBaseException(Exception):
"""Base exception for cvmfsscraper."""

def __init__(
self, message: str, original_excption: Exception = None, *args: Any
) -> None:
def __init__(self, message: str, original_excption: Exception = None, *args: Any) -> None:
"""Initialize the exception."""
self.message = message
self.original_exception = original_excption
Expand Down
16 changes: 4 additions & 12 deletions cvmfsscraper/http_get_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,7 @@ def validate_and_set_host_names(self) -> "GetGeoAPI":
host_names_input = self.host_names_input

if len(host_names_input) != len(self.host_indices):
raise ValueError(
"host_indices and host_names_input must be of the same length."
)
raise ValueError("host_indices and host_names_input must be of the same length.")

return self

Expand Down Expand Up @@ -107,9 +105,7 @@ class GetCVMFSRepositoriesJSON(CVMFSBaseModel):
"populate_by_name": True,
}

schema_version: int = Field(
..., alias="schema", description="The schema version", gt=0
)
schema_version: int = Field(..., alias="schema", description="The schema version", gt=0)
# Stratum0 does not have a last_geodb_update field.
last_geodb_update: Optional[datetime] = Field(
None,
Expand All @@ -125,9 +121,7 @@ class GetCVMFSRepositoriesJSON(CVMFSBaseModel):
None, description="List of repositories"
)

replicas: Optional[List[RepositoryOrReplica]] = Field(
None, description="List of replicas"
)
replicas: Optional[List[RepositoryOrReplica]] = Field(None, description="List of replicas")

@field_validator("last_geodb_update", mode="before")
def convert_cvmfs_date_to_datetime(cls, value: str) -> datetime:
Expand Down Expand Up @@ -268,9 +262,7 @@ def get_catalog_entry(self, name_or_alias: str) -> Any:
try:
return self.model_dump(by_alias=True)[name_or_alias]
except KeyError as exc:
raise AttributeError(
f"No attribute found for alias '{name_or_alias}'"
) from exc
raise AttributeError(f"No attribute found for alias '{name_or_alias}'") from exc

@field_validator(
"root_cryptographic_hash",
Expand Down
4 changes: 1 addition & 3 deletions cvmfsscraper/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,7 @@ def __str__(self) -> str:

def scrape(self) -> None:
"""Scrape the repository."""
log.debug(
"Scraping repository", server=self.server, name=self.name, url=self.path
)
log.debug("Scraping repository", server=self.server, name=self.name, url=self.path)
try:
cvmfspublished = self.fetch_cvmfspublished()
self.parse_cvmfspublished(cvmfspublished)
Expand Down
8 changes: 2 additions & 6 deletions cvmfsscraper/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,7 @@ def populate_repositories(self) -> None:
)
self.fetch_errors.append({"path": self.name, "error": e})

def process_repositories_json(
self, repodata: GetCVMFSRepositoriesJSON
) -> List[Repository]:
def process_repositories_json(self, repodata: GetCVMFSRepositoriesJSON) -> List[Repository]:
"""Process the repositories.json file.
Sets self.repos and self.metadata.
Expand Down Expand Up @@ -267,9 +265,7 @@ def fetch_endpoint(
if not isinstance(endpoint, Endpoints): # type: ignore
raise TypeError("endpoint must be an Endpoints enum value")

log.debug(
"Fetching endpoint", server=self.name, endpoint=endpoint.name, repo=repo
)
log.debug("Fetching endpoint", server=self.name, endpoint=endpoint.name, repo=repo)

geoapi_str = ",".join(geoapi_servers)
formatted_path = endpoint.path.format(repo=repo, geoapi_str=geoapi_str)
Expand Down
4 changes: 1 addition & 3 deletions cvmfsscraper/tests/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,7 @@ def validate_and_load(data_dir: str) -> Dict[str, Union[str, bytes]]:
ENDPOINTS["http://example.com/timeout"] = urllib.error.URLError("timeout")


def mock_urlopen(
url: str, timeout: Union[int, float, None] = None
) -> Union[Mock, Exception]:
def mock_urlopen(url: str, timeout: Union[int, float, None] = None) -> Union[Mock, Exception]:
"""Mock urllib.request.urlopen based on a predefined URL mapping.
:param url: The URL to fetch.
Expand Down
10 changes: 7 additions & 3 deletions cvmfsscraper/tests/test_010_fetch.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Test that fetching data works as expected."""

import json
import sys
from unittest import TestCase
from unittest.mock import Mock, patch

Expand All @@ -19,9 +20,7 @@ class MockedURLLibRequest(TestCase):

def setUp(self) -> None:
"""Mock urllib.request.urlopen."""
self.mock_urlopen = patch(
"urllib.request.urlopen", side_effect=mock_urlopen
).start()
self.mock_urlopen = patch("urllib.request.urlopen", side_effect=mock_urlopen).start()
self.addCleanup(patch.stopall)


Expand All @@ -45,6 +44,11 @@ def test_basic_fetching(self) -> None:

def test_fetching_with_errors(self) -> None:
"""Test fetching data with fetch."""
# structlog fails on this specific test for
# python < 3.10, so skip this test for those versions
if sys.version_info < (3, 10): # pragma: no cover
return

endpoint = "http://stratum1-no.tld/cvmfs/info/v1/repositories.json.404"
obj = Mock()
obj.fetch_errors = []
Expand Down
31 changes: 24 additions & 7 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,13 @@ prometheus_client = "*"
requires = ["poetry-core>=1.7.0"]
build-backend = "poetry.core.masonry.api"

[tool.black]
max-line-length = 99

[tool.coverage.run]
source = ["cvmfsscraper"]

[tool.coverage.report]
fail_under = 100
show_missing = true

[tool.isort]
profile = "black"
line_length = 99

[tool.ruff]
select = [
"A", # flake8-builtins
Expand Down Expand Up @@ -86,8 +79,32 @@ ignore = [
"D213",
# ANN101 (Missing type annotation for `self` in method) is infered.
"ANN101",
# B905 zip() without an explicit strict= parameter set. strict=True causes the resulting iterator
# to raise a ValueError if the arguments are exhausted at differing lengths. Added in python 3.10
# and we support 3.8+.
"B905",

# Rules disabled due to using ruff as a formatter. Most of these rules
# will thus be enforced by the formatter.
# https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
"W191", # Indentation contains tabs
"E111", # Indentation is not a multiple of four
"E114", # Indentation is not a multiple of four (comment)
"E117", # Over-indented
"D206", # Docstring should be indented with spaces, not tabs
"D300", # Use """triple double quotes""" (found '''-quotes)
"Q000", # Remove bad quotes (inlined)
"Q001", # Remove bad quotes (multiline)
"Q002", # Remove bad quotes (docstring)
"Q003", # Avoidable escape quote
"COM812", # Missing trailing comma in Python 3.6+
"COM819", # Prohibited trailing comma
"ISC001", # Implicit string concatenation (single line)
"ISC002", # Implicit string concatenation (multiline)

]


# Allow autofix for all enabled rules (when `--fix`) is provided.
fixable = [
"A",
Expand Down
14 changes: 11 additions & 3 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,21 @@ python =

[testenv:lint]
skip_install = true
description = Invoke black and ruff on the project.
description = Invoke ruff on the project.
allowlist_externals = poetry
commands =
# poetry install -q
poetry run black cvmfsscraper
poetry run ruff format --check cvmfsscraper
poetry run ruff check cvmfsscraper

[testenv:lint-fix]
skip_install = true
description = Invoke ruff on the project, and fix fixable issues.
allowlist_externals = poetry
commands =
poetry run ruff format cvmfsscraper
poetry run ruff check --fix cvmfsscraper


[testenv]
description = Run tests against {envname}.
setenv =
Expand Down

0 comments on commit db1172c

Please sign in to comment.