From 23d3ca2667b1c797305d29e6e8a6d4c7853da8c6 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 30 Oct 2024 10:25:41 +0800 Subject: [PATCH] Add test_sources configuration option. --- cibuildwheel/linux.py | 11 +- cibuildwheel/macos.py | 11 +- cibuildwheel/options.py | 5 + .../resources/cibuildwheel.schema.json | 33 ++++ cibuildwheel/resources/defaults.toml | 1 + .../resources/testing_temp_dir_file.py | 4 +- cibuildwheel/util.py | 39 ++++- cibuildwheel/windows.py | 11 +- docs/options.md | 46 +++++- test/test_testing.py | 20 +++ unit_test/utils_test.py | 144 ++++++++++++++++++ 11 files changed, 315 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 897b5f04f..52628d318 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -20,6 +20,7 @@ from .util import ( BuildFrontendConfig, BuildSelector, + copy_test_sources, find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, @@ -395,7 +396,15 @@ def build_in_container( ) test_cwd = testing_temp_dir / "test_cwd" container.call(["mkdir", "-p", test_cwd]) - container.copy_into(test_fail_cwd_file, test_cwd / "test_fail.py") + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + build_options.package_dir, + test_cwd, + copy_into=container.copy_into, + ) + else: + container.copy_into(test_fail_cwd_file, test_cwd / "test_fail.py") container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f830dbcc6..79ac71f4c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -31,6 +31,7 @@ BuildSelector, call, combine_constraints, + copy_test_sources, detect_ci_provider, download, find_compatible_wheel, @@ -738,7 +739,15 @@ def build(options: Options, tmp_path: Path) -> None: test_cwd = identifier_tmp_dir / "test_cwd" test_cwd.mkdir(exist_ok=True) - (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text()) + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + build_options.package_dir, + test_cwd, + ) + else: + # There are no test sources. Copy the test safety file. + (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text()) shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 5e1e71d16..f1333ed9f 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -90,6 +90,7 @@ class BuildOptions: dependency_constraints: DependencyConstraints | None test_command: str | None before_test: str | None + test_sources: list[str] test_requires: list[str] test_extras: str build_verbosity: int @@ -668,6 +669,9 @@ def build_options(self, identifier: str | None) -> BuildOptions: dependency_versions = self.reader.get("dependency-versions") test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && ")) before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && ")) + test_sources = self.reader.get( + "test-sources", option_format=ListFormat(sep=" ") + ).split() test_requires = self.reader.get( "test-requires", option_format=ListFormat(sep=" ") ).split() @@ -771,6 +775,7 @@ def build_options(self, identifier: str | None) -> BuildOptions: return BuildOptions( globals=self.globals, test_command=test_command, + test_sources=test_sources, test_requires=test_requires, test_extras=test_extras, before_test=before_test, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 8e2508bc9..bd5c170aa 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -397,6 +397,21 @@ ], "title": "CIBW_TEST_EXTRAS" }, + "test-sources": { + "description": "Test files that are required by the test environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_SOURCES" + }, "test-requires": { "description": "Install Python dependencies before running the tests", "oneOf": [ @@ -485,6 +500,9 @@ "test-extras": { "$ref": "#/$defs/inherit" }, + "test-sources": { + "$ref": "#/$defs/inherit" + }, "test-requires": { "$ref": "#/$defs/inherit" } @@ -571,6 +589,9 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, "test-requires": { "$ref": "#/properties/test-requires" } @@ -675,6 +696,9 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, "test-requires": { "$ref": "#/properties/test-requires" } @@ -720,6 +744,9 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, "test-requires": { "$ref": "#/properties/test-requires" } @@ -778,6 +805,9 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, "test-requires": { "$ref": "#/properties/test-requires" } @@ -823,6 +853,9 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, "test-requires": { "$ref": "#/properties/test-requires" } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 21bac7a0c..3073b42d6 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -18,6 +18,7 @@ repair-wheel-command = "" test-command = "" before-test = "" +test-sources = [] test-requires = [] test-extras = [] diff --git a/cibuildwheel/resources/testing_temp_dir_file.py b/cibuildwheel/resources/testing_temp_dir_file.py index 1788e7cf7..d64cda76e 100644 --- a/cibuildwheel/resources/testing_temp_dir_file.py +++ b/cibuildwheel/resources/testing_temp_dir_file.py @@ -13,5 +13,7 @@ def test_fail(self): "wheel. Please specify a path to your tests when invoking pytest " "using the {project} placeholder, e.g. `pytest {project}` or " "`pytest {project}/tests`. cibuildwheel will replace {project} with " - "the path to your project." + "the path to your project. Alternatively, use `test-sources` to " + "specify the files and folders that should be copied into the test " + "environment." ) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index bfbe50c7a..485484231 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -16,7 +16,7 @@ import typing import urllib.request from collections import defaultdict -from collections.abc import Generator, Iterable, Mapping, MutableMapping, Sequence +from collections.abc import Callable, Generator, Iterable, Mapping, MutableMapping, Sequence from dataclasses import dataclass from enum import Enum from functools import lru_cache, total_ordering @@ -35,6 +35,7 @@ from packaging.version import Version from platformdirs import user_cache_path +from . import errors from ._compat import tomllib from .architecture import Architecture from .typing import PathOrStr, PlatformName @@ -394,6 +395,42 @@ def move_file(src_file: Path, dst_file: Path) -> Path: return Path(resulting_file).resolve(strict=True) +def copy_into(src: Path, dst: PurePath) -> None: + """Copy a path from src to dst, regardless of whether it's a file or a directory.""" + # Ensure the target folder location exists + Path(dst.parent).mkdir(exist_ok=True, parents=True) + + if src.is_dir(): + shutil.copytree(src, dst) + else: + shutil.copy(src, dst) + + +def copy_test_sources( + test_sources: list[str], + package_dir: Path, + test_dir: PurePath, + copy_into: Callable[[Path, PurePath], None] = copy_into, +) -> None: + """Copy the list of test sources from the package to the test directory. + + :param test_sources: A list of test paths, relative to the package_dir. + :param package_dir: The root of the package directory. + :param test_dir: The folder where test sources should be placed. + :param copy_info: The copy function to use. By default, does a local + filesystem copy; but an OCIContainer.copy_info method (or equivalent) + can be provided. + """ + for test_path in test_sources: + source = package_dir.resolve() / test_path + + if not source.exists(): + msg = f"Test source {test_path} does not exist." + raise errors.FatalError(msg) + + copy_into(source, test_dir / test_path) + + class DependencyConstraints: def __init__(self, base_file_path: Path): assert base_file_path.exists() diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 449be8109..004eb3751 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -27,6 +27,7 @@ BuildSelector, call, combine_constraints, + copy_test_sources, download, extract_zip, find_compatible_wheel, @@ -574,7 +575,15 @@ def build(options: Options, tmp_path: Path) -> None: ) test_cwd = identifier_tmp_dir / "test_cwd" test_cwd.mkdir() - (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text()) + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + build_options.package_dir, + test_cwd, + ) + else: + # There are no test sources. Copy the test safety file. + (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text()) shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) diff --git a/docs/options.md b/docs/options.md index 37f54bfab..22cd764c1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1410,17 +1410,24 @@ Platform-specific environment variables are also available:
> Execute a shell command to test each built wheel Shell command to run tests after the build. The wheel will be installed -automatically and available for import from the tests. To ensure the wheel is -imported by your tests (instead of your source copy), **tests are not run from -your project directory**. Use the placeholders `{project}` and `{package}` when -specifying paths in your project. If this variable is not set, your wheel will -not be installed after building. +automatically and available for import from the tests. If this variable is not +set, your wheel will not be installed after building. + +To ensure the wheel is imported by your tests (instead of your source copy), +**tests are not run from your project directory** - they are run from a +temporary folder generated by cibuildwheel. To reference test sources, you can +use the placeholders `{project}` and `{package}` in your test command to pass in +the location of your test code: - `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. - `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. +Alternatively, you can use the `CIBW_TEST_SOURCES` setting to pass in the list +of files and folders to copy into the temporary folder. This should *not* +include any code that is included as part of your wheel. + The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
@@ -1535,6 +1542,35 @@ Platform-specific environment variables are also available:
In configuration files, you can use an array, and the items will be joined with `&&`. +### `CIBW_TEST_SOURCES` {: #test-sources} +> Files and folders from the source tree that must be copied into the test environment before running the tests. + +A space-separated list of files and folders, relative to the root of the +project, required for running the tests. + +Platform-specific environment variables are also available:
+`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_PYODIDE` + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Copy the "tests" folder, plus "data/test-image.png" from the source folder to the test folder. + CIBW_TEST_SOURCES: tests data/test-image.png + ``` + +!!! tab examples "pyproject.toml" + + ```toml + # Copy the "tests" folder, plus "data/test-image.png" from the source folder to the test folder. + [tool.cibuildwheel] + test-sources = ["tests", "data/test-image.png"] + ``` + + In configuration files, you can use an array, and the items will be joined with a space. + + ### `CIBW_TEST_REQUIRES` {: #test-requires} > Install Python dependencies before running the tests diff --git a/test/test_testing.py b/test/test_testing.py index a31dae514..7373d2b98 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -184,3 +184,23 @@ def test_bare_pytest_invocation( "Please specify a path to your tests when invoking pytest using the {project} placeholder" in captured.out + captured.err ) + + +def test_test_sources(tmp_path): + project_dir = tmp_path / "project" + project_with_a_test.generate(project_dir) + + # build and test the wheels in the test cwd, after copying in the test sources. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": "pytest", + "CIBW_TEST_COMMAND_WINDOWS": "pytest", + "CIBW_TEST_SOURCES": "test", + }, + ) + + # also check that we got the right wheels + expected_wheels = utils.expected_wheels("spam", "0.1.0") + assert set(actual_wheels) == set(expected_wheels) diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index e3b87be86..c26d9bbf3 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -2,11 +2,14 @@ import textwrap from pathlib import PurePath +from unittest.mock import Mock, call import pytest +from cibuildwheel import errors from cibuildwheel.util import ( FlexibleVersion, + copy_test_sources, find_compatible_wheel, fix_ansi_codes_for_github_actions, format_safe, @@ -221,3 +224,144 @@ def test_flexible_version_comparisons(): assert FlexibleVersion("1.0.1-rhel") > FlexibleVersion("1.0") assert FlexibleVersion("1.0.1-rhel") < FlexibleVersion("1.1") assert FlexibleVersion("1.0.1") == FlexibleVersion("v1.0.1") + + +@pytest.fixture +def sample_project(tmp_path): + """Create a directory structure that contains a range of files.""" + project_path = tmp_path / "project" + + (project_path / "src/deep").mkdir(parents=True) + (project_path / "tests/deep").mkdir(parents=True) + (project_path / "other").mkdir(parents=True) + + (project_path / "pyproject.toml").write_text("A pyproject.toml file") + (project_path / "test.cfg").write_text("A test config file") + + (project_path / "src/__init__.py").write_text("source init") + (project_path / "src/module.py").write_text("source module") + (project_path / "src/deep/__init__.py").write_text("deep source init") + + (project_path / "tests/test_module.py").write_text("test module") + (project_path / "tests/deep/test_module.py").write_text("deep test module") + (project_path / "tests/deep/__init__.py").write_text("deep test init") + + (project_path / "other/module.py").write_text("other module") + + return project_path + + +@pytest.mark.parametrize( + ("test_sources", "expected", "not_expected"), + [ + # Empty test_sources copies nothing. + pytest.param( + [], + [], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + id="empty", + ), + # Single standalone files + pytest.param( + ["pyproject.toml", "tests/deep/test_module.py"], + ["pyproject.toml", "tests/deep/test_module.py"], + [ + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + "tests/deep/__init__.py", + ], + id="single-file", + ), + # A full Directory + pytest.param( + ["tests"], + [ + "tests/test_module.py", + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + ], + id="top-level-directory", + ), + # A partial deep directory + pytest.param( + ["tests/deep"], + [ + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + ], + id="partial-directory", + ), + ], +) +def test_copy_test_sources(tmp_path, sample_project, test_sources, expected, not_expected): + """Test sources can be copied into the test directory.""" + target = tmp_path / "somewhere/test_cwd" + copy_test_sources(test_sources, sample_project, target) + + for path in expected: + assert (tmp_path / "somewhere/test_cwd" / path).is_file() + + for path in not_expected: + assert not (tmp_path / "somewhere/test_cwd" / path).exists() + + +def test_copy_test_sources_missing_file(tmp_path, sample_project): + """If test_sources references a folder that doesn't exist, an error is raised.""" + + with pytest.raises( + errors.FatalError, + match=r"Test source tests/does_not_exist.py does not exist.", + ): + copy_test_sources( + ["pyproject.toml", "tests/does_not_exist.py"], + sample_project, + tmp_path / "somewhere/test_cwd", + ) + + +def test_copy_test_sources_alternate_copy_into(sample_project): + """If an alternate copy_into method is provided, it is used.""" + + target = PurePath("/container/test_cwd") + copy_into = Mock() + + copy_test_sources(["pyproject.toml", "tests"], sample_project, target, copy_into=copy_into) + + copy_into.assert_has_calls( + [ + call(sample_project / "pyproject.toml", target / "pyproject.toml"), + call(sample_project / "tests", target / "tests"), + ], + any_order=True, + )