From 11ef899595bbc07b64ca7fdef85dae80dc4dda08 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 10 Nov 2023 12:09:06 +0100 Subject: [PATCH 1/4] remove collect_upstream_conda_tests and test_modified_upstream --- .github/workflows/tests.yml | 4 - .github/workflows/upstream_tests.yml | 4 - .../collect_upstream_conda_tests.py | 124 -- .../pyproject.toml | 33 - dev/linux/bashrc.sh | 1 - dev/linux/upstream_integration.sh | 1 - dev/linux/upstream_unit.sh | 1 - docs/dev/setup.md | 7 - docs/dev/workflows.md | 3 - tests/test_modified_upstream.py | 1382 ----------------- 10 files changed, 1560 deletions(-) delete mode 100644 dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py delete mode 100644 dev/collect_upstream_conda_tests/pyproject.toml delete mode 100644 tests/test_modified_upstream.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6a74bc5..8d27a693 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,7 +82,6 @@ jobs: --file /opt/conda-libmamba-solver-src/dev/requirements.txt \ --file /opt/conda-libmamba-solver-src/tests/requirements.txt && sudo /opt/conda/bin/python -m pip install /opt/conda-libmamba-solver-src --no-deps -vvv && - sudo /opt/conda/bin/python -m pip install /opt/conda-libmamba-solver-src/dev/collect_upstream_conda_tests/ -vvv && source /opt/conda-src/dev/linux/bashrc.sh && /opt/conda/bin/python -m pytest /opt/conda-libmamba-solver-src -vv -m 'not slow'" @@ -167,7 +166,6 @@ jobs: --file ../conda-libmamba-solver/tests/requirements.txt \ python=${{ matrix.python-version }} conda update openssl ca-certificates certifi - python -m pip install ../conda-libmamba-solver/dev/collect_upstream_conda_tests -vv conda info python -c "from importlib.metadata import version; print('libmambapy', version('libmambapy'))" @@ -263,8 +261,6 @@ jobs: run: | call .\dev-init.bat if errorlevel 1 exit 1 - python -m pip install -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\collect_upstream_conda_tests" - if errorlevel 1 exit 1 python -m pip install --no-deps -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver" if errorlevel 1 exit 1 diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 4e6fae8a..22ec0f62 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -156,9 +156,6 @@ jobs: call .\dev-init.bat if errorlevel 1 exit 1 :: /original logic - :: Install test collection plugin - python -m pip install -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\collect_upstream_conda_tests" - if errorlevel 1 exit 1 :: Install conda-libmamba-solver python -m pip install --no-deps -vv "%GITHUB_WORKSPACE%\conda-libmamba-solver" if errorlevel 1 exit 1 @@ -445,7 +442,6 @@ jobs: ./dev/macos/setup.sh # /original setup python -c "from importlib.metadata import version; print('libmambapy', version('libmambapy'))" - python -m pip install ../conda-libmamba-solver/dev/collect_upstream_conda_tests -vv python -m pip install ../conda-libmamba-solver -vv --no-deps conda info -a # /CONDA-LIBMAMBA-SOLVER CHANGE diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py deleted file mode 100644 index f10fbf86..00000000 --- a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (C) 2022 Anaconda, Inc -# Copyright (C) 2023 conda -# SPDX-License-Identifier: BSD-3-Clause -""" -pytest plugin to modify which upstream (conda/conda) tests are run by pytest. -""" - -# Deselect tests from conda/conda we cannot pass due to different reasons -# These used to be skipped or xfail'd upstream, but we are trying to -# keep it clean from this project's specifics -_deselected_upstream_tests = { - # This test checks for plugin errors and assumes none are present, but - # conda-libmamba-solver counts as one so we need to skip it. - "tests/plugins/test_manager.py": ["test_load_entrypoints_importerror"], - # Conflict report / analysis is done differently with libmamba. - "tests/cli/test_cli_install.py": ["test_find_conflicts_called_once"], - "tests/core/test_solve.py": [ - # SolverStateContainer needed - "test_solve_2", - "test_virtual_package_solver", - "test_broken_install", - # Features / nomkl involved - "test_features_solve_1", - "test_prune_1", - "test_update_prune_2", - "test_update_prune_3", - # Message expected, but libmamba does not report constraints - "test_update_prune_5", - # classic expects implicit update to channel with higher priority, including downgrades - # libmamba does not do this, it just stays in the same channel; should it change? - "test_priority_1", - # FIXME: Known issue: We can use a VERIFY task, but that causes a "dance" across solves, - # where the verification task changes a few specs. Next time it runs it undoes it. - "test_force_remove_1", - # The following are known to fail upstream due to too strict expectations - # We provide the same tests with adjusted checks in tests/test_modified_upstream.py - "test_pinned_1", - "test_freeze_deps_1", - "test_cuda_fail_1", - "test_cuda_fail_2", - "test_update_all_1", - "test_conda_downgrade", - "test_python2_update", - "test_fast_update_with_update_modifier_not_set", - "test_downgrade_python_prevented_with_sane_message", - ], - "tests/test_create.py": [ - # libmamba does not support features - "test_remove_features", - # Known bug in mamba; see https://github.com/mamba-org/mamba/issues/1197 - "test_offline_with_empty_index_cache", - # Adjusted in tests/test_modified_upstream.py - "test_install_features", - # libmamba departs from this behavior in the classic logic - # see https://github.com/conda/conda-libmamba-solver/pull/289 - "test_pinned_override_with_explicit_spec", - # TODO: https://github.com/conda/conda-libmamba-solver/issues/141 - "test_conda_pip_interop_conda_editable_package", - ], - # These use libmamba-incompatible MatchSpecs (name[build_number=1] syntax) - "tests/models/test_prefix_graph.py": [ - "test_deep_cyclical_dependency", - # TODO: Investigate this, since they are solver related-ish - "test_windows_sort_orders_1", - ], - # See https://github.com/conda/conda-libmamba-solver/pull/133#issuecomment-1448607110 - # These failed after enabling the whole unit test suite for `conda/conda`. - # Errors are not critical but would require some further assessment in case fixes are obvious. - "tests/cli/test_main_notices.py": [ - "test_notices_appear_once_when_running_decorated_commands", - "test_notices_does_not_interrupt_command_on_failure", - ], - "tests/conda_env/installers/test_pip.py": [ - "PipInstallerTest::test_stops_on_exception", - "PipInstallerTest::test_straight_install", - ], - "tests/conda_env/specs/test_base.py": [ - "DetectTestCase::test_build_msg", - "DetectTestCase::test_dispatches_to_registered_specs", - "DetectTestCase::test_has_build_msg_function", - "DetectTestCase::test_passes_kwargs_to_all_specs", - "DetectTestCase::test_raises_exception_if_no_detection", - ], - # TODO: Known issue: https://github.com/conda/conda-libmamba-solver/issues/320 - "tests/conda_env/test_cli.py": [ - "test_update_env_no_action_json_output", - "test_update_env_only_pip_json_output", - ], - # TODO: Fix upstream; they seem to assume no other solvers will be active via env var - "tests/plugins/test_solvers.py": [ - "test_get_solver_backend", - "test_get_solver_backend_multiple", - ], - # TODO: Investigate these, since they are solver related-ish - "tests/conda_env/specs/test_requirements.py": [ - "TestRequirements::test_environment", - ], - # Added to test_modified_upstream.py - "tests/test_priority.py": ["test_reorder_channel_priority"], - # Added to test_modified_upstream.py; this passes just by moving it to another test file - "tests/test_misc.py": ["test_explicit_missing_cache_entries"], - # Unrelated to libmamba, but we need to skip it because it fails in CI - "tests/test_activate.py": ["test_bash_basic_integration"], -} - - -def pytest_collection_modifyitems(session, config, items): - """ - We use this hook to modify which upstream tests (from the conda/conda repo) - are run by pytest. - - This hook should not return anything but, instead, modify in place. - """ - selected = [] - deselected = [] - for item in items: - path_key = "/".join(item.path.parts[item.path.parts.index("tests") :]) - item_name_no_brackets = item.name.split("[")[0] - if item_name_no_brackets in _deselected_upstream_tests.get(path_key, []): - deselected.append(item) - continue - selected.append(item) - items[:] = selected - config.hook.pytest_deselected(items=deselected) diff --git a/dev/collect_upstream_conda_tests/pyproject.toml b/dev/collect_upstream_conda_tests/pyproject.toml deleted file mode 100644 index b650d998..00000000 --- a/dev/collect_upstream_conda_tests/pyproject.toml +++ /dev/null @@ -1,33 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "collect-upstream-conda-tests" -version = "0.0.1" -description = "A pytest plugin to filter which upstream tests are run" -authors = [ - {name = "Anaconda, Inc.", email = "conda@continuum.io"} -] -license = {file = "../../LICENSE"} -classifiers = [ - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy" -] -requires-python = ">=3.8" -dependencies = [ - "pytest", -] - -[project.urls] -homepage = "https://github.com/conda/conda-libmamba-solver" - -[project.entry-points.pytest11] -collect-upstream-conda-tests = "collect_upstream_conda_tests" diff --git a/dev/linux/bashrc.sh b/dev/linux/bashrc.sh index fdceb01a..cb04d122 100644 --- a/dev/linux/bashrc.sh +++ b/dev/linux/bashrc.sh @@ -44,7 +44,6 @@ if [ -d "/opt/mamba-src" ]; then fi cd /opt/conda-libmamba-solver-src -sudo /opt/conda/bin/python -m pip install ./dev/collect_upstream_conda_tests/ --no-deps sudo /opt/conda/bin/python -m pip install -e . --no-deps cd /opt/conda-src diff --git a/dev/linux/upstream_integration.sh b/dev/linux/upstream_integration.sh index d614071f..d875629c 100755 --- a/dev/linux/upstream_integration.sh +++ b/dev/linux/upstream_integration.sh @@ -20,7 +20,6 @@ sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repo --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_SRC}/tests/requirements-s3.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" -sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC" --no-deps -vvv # /CONDA LIBMAMBA SOLVER CHANGES eval "$(sudo /opt/conda/bin/python -m conda init --dev bash)" diff --git a/dev/linux/upstream_unit.sh b/dev/linux/upstream_unit.sh index 649a24af..0112ebed 100755 --- a/dev/linux/upstream_unit.sh +++ b/dev/linux/upstream_unit.sh @@ -20,7 +20,6 @@ sudo /opt/conda/bin/conda install --quiet -y --solver=classic --repodata-fn repo --file "${CONDA_SRC}/tests/requirements.txt" \ --file "${CONDA_SRC}/tests/requirements-s3.txt" \ --file "${CONDA_LIBMAMBA_SOLVER_SRC}/dev/requirements.txt" -sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC/dev/collect_upstream_conda_tests/" sudo /opt/conda/bin/python -m pip install "$CONDA_LIBMAMBA_SOLVER_SRC" --no-deps -vvv # /CONDA LIBMAMBA SOLVER CHANGES diff --git a/docs/dev/setup.md b/docs/dev/setup.md index e6c88866..20d3a7aa 100644 --- a/docs/dev/setup.md +++ b/docs/dev/setup.md @@ -77,13 +77,6 @@ $ cd $REPO_LOCATION $ python -m pip install --no-deps -e . ``` -5. Install the test collection plugins (only for upstream tests in `conda/conda`): - -```bash -$ cd $REPO_LOCATION -$ python -m pip install dev/collect_upstream_conda_tests/ -``` - For testing out the `libmamba` solve you can set it several ways: - environment variable `CONDA_SOLVER=libmamba` - pass a flag `--solver=libmamba` diff --git a/docs/dev/workflows.md b/docs/dev/workflows.md index 1bbe9e22..49d52640 100644 --- a/docs/dev/workflows.md +++ b/docs/dev/workflows.md @@ -27,6 +27,3 @@ From the properly mounted `conda/conda` Docker container (see ["Development envi $ cd /opt/conda-src $ CONDA_SOLVER=libmamba pytest ``` - -Note we [deselect some upstream tests in our `pyproject.toml`](../../dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py) for a number of reasons. -For this to work we need to ensure that `pytest` loads that plugin by installing it in the same environment. diff --git a/tests/test_modified_upstream.py b/tests/test_modified_upstream.py deleted file mode 100644 index ea55239c..00000000 --- a/tests/test_modified_upstream.py +++ /dev/null @@ -1,1382 +0,0 @@ -# Copyright (C) 2022 Anaconda, Inc -# Copyright (C) 2023 conda -# SPDX-License-Identifier: BSD-3-Clause -""" -This module fixes some tests found across conda/conda's suite to -check the "spirit" of the test, instead of making explicit comparisons -in stdout messages, overly strict solver checks and other differences -that do not result in incompatible behavior. - -We are copying those offending tests instead of patching them to keep -conda/conda code base as unaffected by this work as possible, but it is -indeed feasible to upgrade those tests in the future for more flexible -comparisons. This is only a workaround during the experimental phase. - -Tests were brought over and patched on Feb 7th, 2022, following the -source found in commit 98fb262c610e17a7731b9183bf37cca98dcc1a71. -""" - -import os -import sys -import warnings -from pprint import pprint - -import pytest -from conda.auxlib.ish import dals -from conda.base.constants import UpdateModifier, on_win -from conda.base.context import conda_tests_ctxt_mgmt_def_pol, context -from conda.common.io import env_var -from conda.core.package_cache_data import PackageCacheData -from conda.core.prefix_data import PrefixData -from conda.core.subdir_data import SubdirData -from conda.exceptions import UnsatisfiableError -from conda.gateways.subprocess import subprocess_call_with_clean_env -from conda.misc import explicit -from conda.models.match_spec import MatchSpec -from conda.models.version import VersionOrder -from conda.testing import ( - CondaCLIFixture, - TmpEnvFixture, - conda_cli, - path_factory, - tmp_env, -) -from conda.testing.cases import BaseTestCase -from conda.testing.helpers import ( - add_subdir, - add_subdir_to_iter, - convert_to_dist_str, - get_solver, - get_solver_2, - get_solver_4, - get_solver_aggregate_1, - get_solver_aggregate_2, - get_solver_cuda, -) -from conda.testing.integration import ( - PYTHON_BINARY, - Commands, - make_temp_env, - package_is_installed, - run_command, -) -from pytest import MonkeyPatch -from pytest_mock import MockerFixture - - -@pytest.mark.integration -class PatchedCondaTestCreate(BaseTestCase): - """ - These tests come from `conda/conda::tests/test_create.py` - """ - - def setUp(self): - PackageCacheData.clear() - - @pytest.mark.xfail( ## MODIFIED - reason="This is not allowed in libmamba: " - "https://github.com/conda/conda-libmamba-solver/pull/289" - ) - def test_pinned_override_with_explicit_spec(self): - with make_temp_env("python=3.8") as prefix: - ## MODIFIED - # Original test assumed the `python=3.6` spec above resolves to `python=3.6.5` - # Instead we only pin whatever the solver decided to install - # Original lines were: - ### run_command(Commands.CONFIG, prefix, - ### "--add", "pinned_packages", "python=3.6.5") - python = next(PrefixData(prefix).query("python")) - run_command( - Commands.CONFIG, prefix, "--add", "pinned_packages", f"python={python.version}" - ) - ## /MODIFIED - - run_command(Commands.INSTALL, prefix, "python=3.7", no_capture=True) - assert package_is_installed(prefix, "python=3.7") - - @pytest.mark.xfail(on_win, reason="TODO: Investigate why this fails on Windows only") - def test_install_update_deps_only_deps_flags(self): - with make_temp_env("flask=2.0.1", "jinja2=3.0.1") as prefix: - python = os.path.join(prefix, PYTHON_BINARY) - result_before = subprocess_call_with_clean_env([python, "--version"]) - assert package_is_installed(prefix, "flask=2.0.1") - assert package_is_installed(prefix, "jinja2=3.0.1") - run_command( - Commands.INSTALL, - prefix, - "flask", - "python", - "--update-deps", - "--only-deps", - no_capture=True, - ) - result_after = subprocess_call_with_clean_env([python, "--version"]) - assert result_before == result_after - assert package_is_installed(prefix, "flask=2.0.1") - assert package_is_installed(prefix, "jinja2>3.0.1") - - -@pytest.mark.xfail(on_win, reason="nomkl not present on windows", strict=True) -def test_install_features(): - # MODIFIED: Added fixture manually - PackageCacheData.clear() - # /MODIFIED - with make_temp_env("python=2", "numpy=1.13", "nomkl", no_capture=True) as prefix: - assert package_is_installed(prefix, "numpy") - assert package_is_installed(prefix, "nomkl") - assert not package_is_installed(prefix, "mkl") - - with make_temp_env("python=2", "numpy=1.13") as prefix: - assert package_is_installed(prefix, "numpy") - assert not package_is_installed(prefix, "nomkl") - assert package_is_installed(prefix, "mkl") - - # run_command(Commands.INSTALL, prefix, "nomkl", no_capture=True) - run_command(Commands.INSTALL, prefix, "python=2", "nomkl", no_capture=True) - # MODIFIED ^: python=2 needed explicitly to trigger update - assert package_is_installed(prefix, "numpy") - assert package_is_installed(prefix, "nomkl") - assert package_is_installed(prefix, "blas=1.0=openblas") - assert not package_is_installed(prefix, "mkl_fft") - assert not package_is_installed(prefix, "mkl_random") - # assert not package_is_installed(prefix, "mkl") # pruned as an indirect dep - - -# The following tests come from `conda/conda::tests/core/test_solve.py` - - -@pytest.mark.integration -def test_pinned_1(tmpdir): - specs = (MatchSpec("numpy"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-3.3.2-0", - "channel-1::numpy-1.7.1-py33_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - with env_var( - "CONDA_PINNED_PACKAGES", - "python=2.6&iopro<=1.4.2", - stack_callback=conda_tests_ctxt_mgmt_def_pol, - ): - specs = (MatchSpec("system=5.8=0"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter(("channel-1::system-5.8-0",)) - assert convert_to_dist_str(final_state_1) == order - - # ignore_pinned=True - specs_to_add = (MatchSpec("python"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(ignore_pinned=True) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-3.3.2-0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - # ignore_pinned=False - specs_to_add = (MatchSpec("python"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(ignore_pinned=False) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - # incompatible CLI and configured specs - specs_to_add = (MatchSpec("scikit-learn==0.13"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - ## MODIFIED - # Original tests checks for SpecsConfigurationConflictError being raised - # but libmamba will fails with UnsatisfiableError instead. Hence, we check - # the error string. Original check inspected the kwargs of the exception: - ### with pytest.raises(SpecsConfigurationConflictError) as exc: - ### solver.solve_final_state(ignore_pinned=False) - ### kwargs = exc.value._kwargs - ### assert kwargs["requested_specs"] == ["scikit-learn==0.13"] - ### assert kwargs["pinned_specs"] == ["python=2.6"] - with pytest.raises(UnsatisfiableError) as exc_info: - solver.solve_final_state(ignore_pinned=False) - error = str(exc_info.value) - assert "package scikit-learn-0.13" in error - assert "requires python 2.7*" in error - ## /MODIFIED - - specs_to_add = (MatchSpec("numba"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_2, - history_specs=history_specs, - ) as solver: - final_state_3 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_3)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_3) == order - - specs_to_add = (MatchSpec("python"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_3, - history_specs=history_specs, - ) as solver: - final_state_4 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_DEPS) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_4)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_4) == order - - specs_to_add = (MatchSpec("python"),) - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_4, - history_specs=history_specs, - ) as solver: - final_state_5 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_5)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::argparse-1.2.1-py26_0", - "channel-1::llvmpy-0.11.2-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.8.1-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_5) == order - - # now update without pinning - # MODIFIED: libmamba decides to stay in python=2.6 unless explicit - # specs_to_add = (MatchSpec("python"),) - specs_to_add = (MatchSpec("python=3"),) - # /MODIFIED - history_specs = ( - MatchSpec("python"), - MatchSpec("system=5.8=0"), - MatchSpec("numba"), - ) - with get_solver( - tmpdir, - specs_to_add=specs_to_add, - prefix_records=final_state_4, - history_specs=history_specs, - ) as solver: - final_state_5 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_1, specs).open_url() - print(convert_to_dist_str(final_state_5)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-3.3.2-0", - "channel-1::llvmpy-0.11.2-py33_0", - "channel-1::numpy-1.7.1-py33_0", - "channel-1::numba-0.8.1-np17py33_0", - ) - ) - assert convert_to_dist_str(final_state_5) == order - - -@pytest.mark.integration -def test_freeze_deps_1(tmpdir): - specs = (MatchSpec("six=1.7"),) - with get_solver_2(tmpdir, specs) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::xz-5.2.3-0", - "channel-2::zlib-1.2.11-0", - "channel-2::python-3.4.5-0", - "channel-2::six-1.7.3-py34_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - specs_to_add = (MatchSpec("bokeh"),) - with get_solver_2( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = () - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::backports_abc-0.5-py34_0", - "channel-2::markupsafe-1.0-py34_0", - "channel-2::numpy-1.13.0-py34_0", - "channel-2::pyyaml-3.12-py34_0", - "channel-2::requests-2.14.2-py34_0", - "channel-2::setuptools-27.2.0-py34_0", - "channel-2::jinja2-2.9.6-py34_0", - "channel-2::python-dateutil-2.6.1-py34_0", - "channel-2::tornado-4.4.2-py34_0", - "channel-2::bokeh-0.12.4-py34_0", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # now we can't install the latest bokeh 0.12.5, but instead we get bokeh 0.12.4 - specs_to_add = (MatchSpec("bokeh"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = () - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::backports_abc-0.5-py34_0", - "channel-2::markupsafe-1.0-py34_0", - "channel-2::numpy-1.13.0-py34_0", - "channel-2::pyyaml-3.12-py34_0", - "channel-2::requests-2.14.2-py34_0", - "channel-2::setuptools-27.2.0-py34_0", - "channel-2::jinja2-2.9.6-py34_0", - "channel-2::python-dateutil-2.6.1-py34_0", - "channel-2::tornado-4.4.2-py34_0", - "channel-2::bokeh-0.12.4-py34_0", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # here, the python=3.4 spec can't be satisfied, so it's dropped, and we go back to py27 - with pytest.raises(UnsatisfiableError): - specs_to_add = (MatchSpec("bokeh=0.12.5"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - unlink_precs, link_precs = solver.solve_for_diff() - - # adding the explicit python spec allows conda to change the python versions. - # one possible outcome is that this updates to python 3.6. That is not desirable because of the - # explicit "six=1.7" request in the history. It should only neuter that spec if there's no way - # to solve it with that spec. - specs_to_add = MatchSpec("bokeh=0.12.5"), MatchSpec("python") - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - ## ADDED - solver._command = "install" - ## /ADDED - - unlink_precs, link_precs = solver.solve_for_diff() - - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-2::six-1.7.3-py34_0", - "channel-2::python-3.4.5-0", - # MODIFIED: xz is not uninstalled for some reason in libmamba :shrug: - # "channel-2::xz-5.2.3-0", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::python-2.7.13-0", - "channel-2::backports-1.0-py27_0", - "channel-2::backports_abc-0.5-py27_0", - "channel-2::certifi-2016.2.28-py27_0", - "channel-2::futures-3.1.1-py27_0", - "channel-2::markupsafe-1.0-py27_0", - "channel-2::numpy-1.13.1-py27_0", - "channel-2::pyyaml-3.12-py27_0", - "channel-2::requests-2.14.2-py27_0", - "channel-2::six-1.7.3-py27_0", - "channel-2::python-dateutil-2.6.1-py27_0", - "channel-2::setuptools-36.4.0-py27_1", - "channel-2::singledispatch-3.4.0.3-py27_0", - "channel-2::ssl_match_hostname-3.5.0.1-py27_0", - "channel-2::jinja2-2.9.6-py27_0", - "channel-2::tornado-4.5.2-py27_0", - "channel-2::bokeh-0.12.5-py27_1", - ) - ) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - # here, the python=3.4 spec can't be satisfied, so it's dropped, and we go back to py27 - specs_to_add = (MatchSpec("bokeh=0.12.5"),) - with get_solver_2( - tmpdir, - specs_to_add, - prefix_records=final_state_1, - history_specs=(MatchSpec("six=1.7"), MatchSpec("python=3.4")), - ) as solver: - with pytest.raises(UnsatisfiableError): - ## ADDED - solver._command = "install" - ## /ADDED - solver.solve_final_state(update_modifier=UpdateModifier.FREEZE_INSTALLED) - - -def test_cuda_fail_1(tmpdir): - specs = (MatchSpec("cudatoolkit"),) - - # No cudatoolkit in index for CUDA 8.0 - with env_var("CONDA_OVERRIDE_CUDA", "8.0"): - with get_solver_cuda(tmpdir, specs) as solver: - with pytest.raises(UnsatisfiableError) as exc: - final_state = solver.solve_final_state() - - ## MODIFIED - # libmamba will generate a slightly different error message, but the spirit is the same. - # Original check was: - ### if sys.platform == "darwin": - ### plat = "osx-64" - ### elif sys.platform == "linux": - ### plat = "linux-64" - ### elif sys.platform == "win32": - ### if platform.architecture()[0] == "32bit": - ### plat = "win-32" - ### else: - ### plat = "win-64" - ### else: - ### plat = "linux-64" - ### assert str(exc.value).strip() == dals("""The following specifications were found to be incompatible with your system: - ### - ### - feature:/{}::__cuda==8.0=0 - ### - cudatoolkit -> __cuda[version='>=10.0|>=9.0'] - ### - ### Your installed version is: 8.0""".format(plat)) - possible_messages = [ - dals( - """Encountered problems while solving: - - nothing provides __cuda >=9.0 needed by cudatoolkit-9.0-0""" - ), - dals( - """Encountered problems while solving: - - nothing provides __cuda >=10.0 needed by cudatoolkit-10.0-0""" - ), - ] - exc_msg = str(exc.value).strip() - assert any(msg in exc_msg for msg in possible_messages) - ## /MODIFIED - - -def test_cuda_fail_2(tmpdir): - specs = (MatchSpec("cudatoolkit"),) - - # No CUDA on system - with env_var("CONDA_OVERRIDE_CUDA", ""): - with get_solver_cuda(tmpdir, specs) as solver: - with pytest.raises(UnsatisfiableError) as exc: - final_state = solver.solve_final_state() - - ## MODIFIED - # libmamba will generate a slightly different error message, but the spirit is the same. - # Original check was: - ### assert str(exc.value).strip() == dals("""The following specifications were found to be incompatible with your system: - ### - ### - cudatoolkit -> __cuda[version='>=10.0|>=9.0'] - ### - ### Your installed version is: not available""") - possible_messages = [ - dals( - """Encountered problems while solving: - - nothing provides __cuda >=9.0 needed by cudatoolkit-9.0-0""" - ), - dals( - """Encountered problems while solving: - - nothing provides __cuda >=10.0 needed by cudatoolkit-10.0-0""" - ), - ] - exc_msg = str(exc.value).strip() - assert any(msg in exc_msg for msg in possible_messages) - ## /MODIFIED - - -def test_update_all_1(tmpdir): - ## MODIFIED - # Libmamba requires MatchSpec.conda_build_form() internally, which depends on `version` and - # `build` fields. `system` below is using only `build_number`, so we have to adapt the syntax - # accordingly. It should be the same result, but in a conda_build_form-friendly way: - ### specs = MatchSpec("numpy=1.5"), MatchSpec("python=2.6"), MatchSpec("system[build_number=0]") - specs = ( - MatchSpec("numpy=1.5"), - MatchSpec("python=2.6"), - MatchSpec("system[version=*,build=*0]"), - ) - ## /MODIFIED - - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - print(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - "channel-1::numpy-1.5.1-py26_4", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - specs_to_add = MatchSpec("numba=0.6"), MatchSpec("numpy") - with get_solver( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - # PrefixDag(final_state_2, specs).open_url() - print(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-0", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", - "channel-1::llvmpy-0.10.2-py26_0", - "channel-1::nose-1.3.0-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.6.0-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - specs_to_add = (MatchSpec("numba=0.6"),) - with get_solver( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state(update_modifier=UpdateModifier.UPDATE_ALL) - # PrefixDag(final_state_2, specs).open_url() - print(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::llvm-3.2-0", - "channel-1::python-2.6.8-6", # stick with python=2.6 even though UPDATE_ALL - "channel-1::llvmpy-0.10.2-py26_0", - "channel-1::nose-1.3.0-py26_0", - "channel-1::numpy-1.7.1-py26_0", - "channel-1::numba-0.6.0-np17py26_0", - ) - ) - assert convert_to_dist_str(final_state_2) == order - - -def test_conda_downgrade(tmpdir): - specs = (MatchSpec("conda-build"),) - with env_var("CONDA_CHANNEL_PRIORITY", "False", stack_callback=conda_tests_ctxt_mgmt_def_pol): - with get_solver_aggregate_1(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-2::conda-env-2.6.0-0", - "channel-2::libffi-3.2.1-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-2::zlib-1.2.11-0", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::patchelf-0.9-hf484d3e_2", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::beautifulsoup4-4.6.3-py37_0", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::cryptography-vectors-2.3-py37_0", - "channel-4::filelock-3.0.4-py37_0", - "channel-4::glob2-0.6-py37_0", - "channel-4::idna-2.7-py37_0", - "channel-4::markupsafe-1.0-py37h14c3975_1", - "channel-4::pkginfo-1.4.2-py37_1", - "channel-4::psutil-5.4.6-py37h14c3975_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::pyyaml-3.13-py37h14c3975_0", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::setuptools-40.0.0-py37_0", - "channel-4::cryptography-2.3-py37hb7f436b_0", - "channel-4::jinja2-2.10-py37_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::conda-build-3.12.1-py37_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - SubdirData.clear_cached_local_channel_data() - specs_to_add = (MatchSpec("itsdangerous"),) # MatchSpec("conda"), - saved_sys_prefix = sys.prefix - try: - sys.prefix = tmpdir.strpath - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = ( - # no conda downgrade - ) - link_order = add_subdir_to_iter(("channel-2::itsdangerous-0.24-py_0",)) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - specs_to_add = ( - MatchSpec("itsdangerous"), - MatchSpec("conda"), - ) - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - assert convert_to_dist_str(unlink_precs) == unlink_order - assert convert_to_dist_str(link_precs) == link_order - - specs_to_add = MatchSpec("itsdangerous"), MatchSpec("conda<4.4.10"), MatchSpec("python") - with get_solver_aggregate_1( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - # now conda gets downgraded - "channel-4::conda-build-3.12.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::jinja2-2.10-py37_0", - "channel-4::cryptography-2.3-py37hb7f436b_0", - "channel-4::setuptools-40.0.0-py37_0", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::pyyaml-3.13-py37h14c3975_0", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::psutil-5.4.6-py37h14c3975_0", - "channel-4::pkginfo-1.4.2-py37_1", - "channel-4::markupsafe-1.0-py37h14c3975_1", - "channel-4::idna-2.7-py37_0", - "channel-4::glob2-0.6-py37_0", - "channel-4::filelock-3.0.4-py37_0", - "channel-4::cryptography-vectors-2.3-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::beautifulsoup4-4.6.3-py37_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::ncurses-6.1-hf484d3e_0", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::xz-5.2.3-0", - "channel-2::yaml-0.1.6-0", - "channel-2::python-3.6.2-0", - "channel-2::asn1crypto-0.22.0-py36_0", - "channel-4::beautifulsoup4-4.6.3-py36_0", - "channel-2::certifi-2016.2.28-py36_0", - "channel-4::chardet-3.0.4-py36_1", - "channel-4::filelock-3.0.4-py36_0", - "channel-4::glob2-0.6-py36_0", - "channel-2::idna-2.6-py36_0", - "channel-2::itsdangerous-0.24-py36_0", - "channel-2::markupsafe-1.0-py36_0", - "channel-4::pkginfo-1.4.2-py36_1", - "channel-2::psutil-5.2.2-py36_0", - "channel-2::pycosat-0.6.2-py36_0", - "channel-2::pycparser-2.18-py36_0", - "channel-2::pyparsing-2.2.0-py36_0", - "channel-2::pyyaml-3.12-py36_0", - "channel-2::requests-2.14.2-py36_0", - "channel-2::ruamel_yaml-0.11.14-py36_1", - "channel-2::six-1.10.0-py36_0", - "channel-2::cffi-1.10.0-py36_0", - "channel-2::packaging-16.8-py36_0", - "channel-2::setuptools-36.4.0-py36_1", - "channel-2::cryptography-1.8.1-py36_0", - "channel-2::jinja2-2.9.6-py36_0", - "channel-2::pyopenssl-17.0.0-py36_0", - "channel-2::conda-4.3.30-py36h5d9f9f4_0", - "channel-4::conda-build-3.12.1-py36_0", - ) - ) - ## MODIFIED - # Original checks verified the full solution was strictly matched: - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - # We only check for conda itself and the explicit specs - # The other packages are slightly different; - # again libedit and ncurses are involved - # (they are also involved in test_fast_update_with_update_modifier_not_set) - for pkg in link_precs: - if pkg.name == "conda": - assert VersionOrder(pkg.version) < VersionOrder("4.4.10") - # TODO: these assertions are a bit flaky (only true in some attempts) - # to be fixed at https://github.com/conda/conda-libmamba-solver/issues/317 - # elif pkg.name == "python": - # assert pkg.version == "3.6.2" - # elif pkg.name == "conda-build": - # assert pkg.version == "3.12.1" - # elif pkg.name == "itsdangerous": - # assert pkg.version == "0.24" - ## /MODIFIED - finally: - sys.prefix = saved_sys_prefix - - -def test_python2_update(tmpdir): - # Here we're actually testing that a user-request will uninstall incompatible packages - # as necessary. - specs = MatchSpec("conda"), MatchSpec("python=2") - with get_solver_4(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order1 = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::conda-env-2.6.0-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-2.7.15-h1571d57_0", - "channel-4::asn1crypto-0.24.0-py27_0", - "channel-4::certifi-2018.8.13-py27_0", - "channel-4::chardet-3.0.4-py27_1", - "channel-4::cryptography-vectors-2.3-py27_0", - "channel-4::enum34-1.1.6-py27_1", - "channel-4::futures-3.2.0-py27_0", - "channel-4::idna-2.7-py27_0", - "channel-4::ipaddress-1.0.22-py27_0", - "channel-4::pycosat-0.6.3-py27h14c3975_0", - "channel-4::pycparser-2.18-py27_1", - "channel-4::pysocks-1.6.8-py27_0", - "channel-4::ruamel_yaml-0.15.46-py27h14c3975_0", - "channel-4::six-1.11.0-py27_1", - "channel-4::cffi-1.11.5-py27h9745a5d_0", - "channel-4::cryptography-2.3-py27hb7f436b_0", - "channel-4::pyopenssl-18.0.0-py27_0", - "channel-4::urllib3-1.23-py27_0", - "channel-4::requests-2.19.1-py27_0", - "channel-4::conda-4.5.10-py27_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order1 - - specs_to_add = (MatchSpec("python=3"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_2)) - order = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::conda-env-2.6.0-1", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::yaml-0.1.7-had09818_2", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.24.0-h84994c4_0", - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::asn1crypto-0.24.0-py37_0", - "channel-4::certifi-2018.8.13-py37_0", - "channel-4::chardet-3.0.4-py37_1", - "channel-4::idna-2.7-py37_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - "channel-4::pycparser-2.18-py37_1", - "channel-4::pysocks-1.6.8-py37_0", - "channel-4::ruamel_yaml-0.15.46-py37h14c3975_0", - "channel-4::six-1.11.0-py37_1", - "channel-4::cffi-1.11.5-py37h9745a5d_0", - "channel-4::cryptography-2.2.2-py37h14c3975_0", - "channel-4::pyopenssl-18.0.0-py37_0", - "channel-4::urllib3-1.23-py37_0", - "channel-4::requests-2.19.1-py37_0", - "channel-4::conda-4.5.10-py37_0", - ) - ) - - ## MODIFIED - # libmamba has a different solution here (cryptography 2.3 instead of 2.2.2) - # and cryptography-vectors (not present in regular conda) - # they are essentially the same functional solution; the important part here - # is that the env migrated to Python 3.7, so we only check some packages - # Original check: - ### assert convert_to_dist_str(final_state_2) == order - full_solution = convert_to_dist_str(final_state_2) - important_parts = add_subdir_to_iter( - ( - "channel-4::python-3.7.0-hc3d631a_0", - "channel-4::conda-4.5.10-py37_0", - "channel-4::pycosat-0.6.3-py37h14c3975_0", - ) - ) - assert set(important_parts).issubset(set(full_solution)) - ## /MODIFIED - - -def test_fast_update_with_update_modifier_not_set(tmpdir): - specs = ( - MatchSpec("python=2"), - MatchSpec("openssl==1.0.2l"), - MatchSpec("sqlite=3.21"), - ) - with get_solver_4(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order1 = add_subdir_to_iter( - ( - "channel-4::ca-certificates-2018.03.07-0", - "channel-4::libgcc-ng-8.2.0-hdf63c60_0", - "channel-4::libstdcxx-ng-8.2.0-hdf63c60_0", - "channel-4::libffi-3.2.1-hd88cf55_4", - "channel-4::ncurses-6.0-h9df7e31_2", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::tk-8.6.7-hc745277_3", - "channel-4::zlib-1.2.11-ha838bed_2", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::readline-7.0-ha6073c6_4", - "channel-4::sqlite-3.21.0-h1bed415_2", - "channel-4::python-2.7.14-h89e7a4a_22", - ) - ) - assert convert_to_dist_str(final_state_1) == order1 - - specs_to_add = (MatchSpec("python"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-4::python-2.7.14-h89e7a4a_22", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::ncurses-6.0-h9df7e31_2", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::xz-5.2.4-h14c3975_4", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::python-3.6.4-hc3d631a_1", # python is upgraded - ) - ) - ## MODIFIED - # We only check python was upgraded as expected, not the full solution - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - assert add_subdir("channel-4::python-2.7.14-h89e7a4a_22") in convert_to_dist_str( - unlink_precs - ) - assert add_subdir("channel-4::python-3.6.4-hc3d631a_1") in convert_to_dist_str(link_precs) - ## /MODIFIED - - specs_to_add = (MatchSpec("sqlite"),) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - unlink_precs, link_precs = solver.solve_for_diff() - pprint(convert_to_dist_str(unlink_precs)) - pprint(convert_to_dist_str(link_precs)) - unlink_order = add_subdir_to_iter( - ( - "channel-4::python-2.7.14-h89e7a4a_22", - "channel-4::sqlite-3.21.0-h1bed415_2", - "channel-4::libedit-3.1-heed3624_0", - "channel-4::openssl-1.0.2l-h077ae2c_5", - "channel-4::ncurses-6.0-h9df7e31_2", - ) - ) - link_order = add_subdir_to_iter( - ( - "channel-4::ncurses-6.1-hf484d3e_0", - "channel-4::openssl-1.0.2p-h14c3975_0", - "channel-4::libedit-3.1.20170329-h6b74fdf_2", - "channel-4::sqlite-3.24.0-h84994c4_0", # sqlite is upgraded - "channel-4::python-2.7.15-h1571d57_0", # python is not upgraded - ) - ) - ## MODIFIED - # We only check sqlite was upgraded as expected and python stays the same - ### assert convert_to_dist_str(unlink_precs) == unlink_order - ### assert convert_to_dist_str(link_precs) == link_order - assert add_subdir("channel-4::sqlite-3.21.0-h1bed415_2") in convert_to_dist_str( - unlink_precs - ) - sqlite = next(pkg for pkg in link_precs if pkg.name == "sqlite") - # mamba chooses a different sqlite version (3.23 instead of 3.24) - assert VersionOrder(sqlite.version) > VersionOrder("3.21") - # If Python was changed, it should have stayed at 2.7 - python = next((pkg for pkg in link_precs if pkg.name == "python"), None) - if python: - assert python.version.startswith("2.7") - ## /MODIFIED - - specs_to_add = ( - MatchSpec("sqlite"), - MatchSpec("python"), - ) - with get_solver_4( - tmpdir, specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state( - update_modifier=UpdateModifier.SPECS_SATISFIED_SKIP_SOLVE - ) - pprint(convert_to_dist_str(final_state_2)) - assert convert_to_dist_str(final_state_2) == order1 - - -@pytest.mark.xfail(True, reason="Known bug: mamba prefers arch to noarch - TODO") -def test_channel_priority_churn_minimized(tmpdir): - specs = ( - MatchSpec("conda-build"), - MatchSpec("itsdangerous"), - ) - with get_solver_aggregate_2(tmpdir, specs) as solver: - final_state = solver.solve_final_state() - - pprint(convert_to_dist_str(final_state)) - - with get_solver_aggregate_2( - tmpdir, [MatchSpec("itsdangerous")], prefix_records=final_state, history_specs=specs - ) as solver: - solver.channels.reverse() - unlink_dists, link_dists = solver.solve_for_diff( - update_modifier=UpdateModifier.FREEZE_INSTALLED - ) - pprint(convert_to_dist_str(unlink_dists)) - pprint(convert_to_dist_str(link_dists)) - assert len(unlink_dists) == 1 - assert len(link_dists) == 1 - - -@pytest.mark.xfail(True, reason="channel priority is a bit different in libmamba; TODO") -def test_priority_1(tmpdir): - with env_var("CONDA_SUBDIR", "linux-64", stack_callback=conda_tests_ctxt_mgmt_def_pol): - specs = ( - MatchSpec("pandas"), - MatchSpec("python=2.7"), - ) - - ## MODIFIED - # Original value was set to True (legacy value for "flexible" nowadays), but libmamba - # only gets the same solution is strict priority is chosen. It _looks_ like this was the - # intention of the test anyways, but it should be investigated further. Marking as xfail for now. - ### with env_var("CONDA_CHANNEL_PRIORITY", "True", stack_callback=conda_tests_ctxt_mgmt_def_pol): - with env_var( - "CONDA_CHANNEL_PRIORITY", "strict", stack_callback=conda_tests_ctxt_mgmt_def_pol - ): - ## /MODIFIED - - with get_solver_aggregate_1(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-2::mkl-2017.0.3-0", - "channel-2::openssl-1.0.2l-0", - "channel-2::readline-6.2-2", - "channel-2::sqlite-3.13.0-0", - "channel-2::tk-8.5.18-0", - "channel-2::zlib-1.2.11-0", - "channel-2::python-2.7.13-0", - "channel-2::numpy-1.13.1-py27_0", - "channel-2::pytz-2017.2-py27_0", - "channel-2::six-1.10.0-py27_0", - "channel-2::python-dateutil-2.6.1-py27_0", - "channel-2::pandas-0.20.3-py27_0", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - with env_var( - "CONDA_CHANNEL_PRIORITY", "False", stack_callback=conda_tests_ctxt_mgmt_def_pol - ): - with get_solver_aggregate_1( - tmpdir, specs, prefix_records=final_state_1, history_specs=specs - ) as solver: - final_state_2 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_2)) - # python and pandas will be updated as they are explicit specs. Other stuff may or may not, - # as required to satisfy python and pandas - order = add_subdir_to_iter( - ( - "channel-4::python-2.7.15-h1571d57_0", - "channel-4::pandas-0.23.4-py27h04863e7_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_2) - - # channel priority taking effect here. channel-2 should be the channel to draw from. Downgrades expected. - # python and pandas will be updated as they are explicit specs. Other stuff may or may not, - # as required to satisfy python and pandas - with get_solver_aggregate_1( - tmpdir, specs, prefix_records=final_state_2, history_specs=specs - ) as solver: - final_state_3 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_3)) - order = add_subdir_to_iter( - ( - "channel-2::python-2.7.13-0", - "channel-2::pandas-0.20.3-py27_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_3) - - specs_to_add = (MatchSpec("six<1.10"),) - specs_to_remove = (MatchSpec("pytz"),) - with get_solver_aggregate_1( - tmpdir, - specs_to_add=specs_to_add, - specs_to_remove=specs_to_remove, - prefix_records=final_state_3, - history_specs=specs, - ) as solver: - final_state_4 = solver.solve_final_state() - pprint(convert_to_dist_str(final_state_4)) - order = add_subdir_to_iter( - ( - "channel-2::python-2.7.13-0", - "channel-2::six-1.9.0-py27_0", - ) - ) - for spec in order: - assert spec in convert_to_dist_str(final_state_4) - assert "pandas" not in convert_to_dist_str(final_state_4) - - -def test_downgrade_python_prevented_with_sane_message(tmpdir): - specs = (MatchSpec("python=2.6"),) - with get_solver(tmpdir, specs) as solver: - final_state_1 = solver.solve_final_state() - # PrefixDag(final_state_1, specs).open_url() - pprint(convert_to_dist_str(final_state_1)) - order = add_subdir_to_iter( - ( - "channel-1::openssl-1.0.1c-0", - "channel-1::readline-6.2-0", - "channel-1::sqlite-3.7.13-0", - "channel-1::system-5.8-1", - "channel-1::tk-8.5.13-0", - "channel-1::zlib-1.2.7-0", - "channel-1::python-2.6.8-6", - ) - ) - assert convert_to_dist_str(final_state_1) == order - - # incompatible CLI and configured specs - specs_to_add = (MatchSpec("scikit-learn==0.13"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - with pytest.raises(UnsatisfiableError) as exc: - solver.solve_final_state() - - error_msg = str(exc.value).strip() - - ## MODIFIED - # One more case of different wording for the same message. I think the essence is the same - # (cannot update to python 2.7), even if python 2.6 is not mentioned. - ### assert "incompatible with the existing python installation in your environment:" in error_msg - ### assert "- scikit-learn==0.13 -> python=2.7" in error_msg - ### assert "Your python: python=2.6" in error_msg - assert "Encountered problems while solving" in error_msg - assert "package scikit-learn-0.13" in error_msg and "requires python 2.7*" in error_msg - ## /MODIFIED - - specs_to_add = (MatchSpec("unsatisfiable-with-py26"),) - with get_solver( - tmpdir, specs_to_add=specs_to_add, prefix_records=final_state_1, history_specs=specs - ) as solver: - with pytest.raises(UnsatisfiableError) as exc: - solver.solve_final_state() - error_msg = str(exc.value).strip() - - ## MODIFIED - # In this case, the error is not as similar! We are still accepting it, but it could use - # some improvements... Note how Python is not mentioned at all, just scikit-learn. - # Leaving a # TODO mark here so we can come revisit this in the future. - ### assert "incompatible with the existing python installation in your environment:" in error_msg - ### assert "- unsatisfiable-with-py26 -> python=2.7" in error_msg - ### assert "Your python: python=2.6" - assert "Encountered problems while solving" in error_msg - assert "package unsatisfiable-with-py26-1.0-0 requires scikit-learn 0.13" in error_msg - ## /MODIFIED - - -# The following tests come from tests/test_priority.py - - -@pytest.mark.integration -@pytest.mark.parametrize( - "pinned_package", - [ - pytest.param(True, id="with pinned_package"), - pytest.param(False, id="without pinned_package"), - ], -) -def test_reorder_channel_priority( - tmp_env: TmpEnvFixture, - monkeypatch: MonkeyPatch, - conda_cli: CondaCLIFixture, - pinned_package: bool, -): - # use "cheap" packages with no dependencies - package1 = "zlib" - package2 = "ca-certificates" - - # set pinned package - if pinned_package: - monkeypatch.setenv("CONDA_PINNED_PACKAGES", package1) - - # create environment with package1 and package2 - with tmp_env("--override-channels", "--channel=defaults", package1, package2) as prefix: - # check both packages are installed from defaults - PrefixData._cache_.clear() - assert PrefixData(prefix).get(package1).channel.name == "pkgs/main" - assert PrefixData(prefix).get(package2).channel.name == "pkgs/main" - - # update --all - out, err, retcode = conda_cli( - "update", - f"--prefix={prefix}", - "--override-channels", - "--channel=conda-forge", - "--all", - "--yes", - ) - # check pinned package is unchanged but unpinned packages are updated from conda-forge - PrefixData._cache_.clear() - expected_channel = "pkgs/main" if pinned_package else "conda-forge" - assert PrefixData(prefix).get(package1).channel.name == expected_channel - # assert PrefixData(prefix).get(package2).channel.name == "conda-forge" - # MODIFIED ^: Some packages do not change channels in libmamba - - -def test_explicit_missing_cache_entries( - mocker: MockerFixture, - conda_cli: CondaCLIFixture, - tmp_env: TmpEnvFixture, -): - """Test that explicit() raises and notifies if some of the specs were not found in the cache.""" - from conda.core.package_cache_data import PackageCacheData - - with tmp_env() as prefix: # ensure writable env - if len(PackageCacheData.get_all_extracted_entries()) == 0: - # Package cache e.g. ./devenv/Darwin/x86_64/envs/devenv-3.9-c/pkgs/ can - # be empty in certain cases (Noted in OSX with Python 3.9, when - # Miniconda installs Python 3.10). Install a small package. - warnings.warn("test_explicit_missing_cache_entries: No packages in cache.") - out, err, retcode = conda_cli("install", "--prefix", prefix, "heapdict", "--yes") - assert retcode == 0, (out, err) # MODIFIED - - # Patching ProgressiveFetchExtract prevents trying to download a package from the url. - # Note that we cannot monkeypatch context.dry_run, because explicit() would exit early with that. - mocker.patch("conda.misc.ProgressiveFetchExtract") - print(PackageCacheData.get_all_extracted_entries()[0]) # MODIFIED - with pytest.raises( - AssertionError, - match="Missing package cache records for: pkgs/linux-64::foo==1.0.0=py_0", - ): - explicit( - [ - "http://test/pkgs/linux-64/foo-1.0.0-py_0.tar.bz2", # does not exist - PackageCacheData.get_all_extracted_entries()[0].url, # exists - ], - prefix, - ) From a73de0c11aa9bb6bdfced9cefb14a3444ce9a425 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 10 Nov 2023 12:23:31 +0100 Subject: [PATCH 2/4] sync workflow with upstream --- .github/workflows/upstream_tests.yml | 235 +++++++++------------------ 1 file changed, 76 insertions(+), 159 deletions(-) diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 22ec0f62..4a73f1ef 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -4,20 +4,19 @@ name: Upstream tests # CONDA-LIBMAMBA-SOLVER CHANGE on: - # NOTE: github.event context is push payload: - # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push push: branches: - main - feature/** + - '[0-9].*.x' # e.g., 4.14.x + - '[0-9][0-9].*.x' # e.g., 23.3.x - # NOTE: github.event context is pull_request payload: - # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request pull_request: - # CONDA-LIBMAMBA-SOLVER CHANGE - schedule: - - cron: "15 7 * * 1-5" # Mon to Fri, 7:15am - # /CONDA-LIBMAMBA-SOLVER CHANGE + + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch + workflow_dispatch: concurrency: # Concurrency group that uses the workflow name and PR number if available @@ -28,29 +27,24 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true -env: - # see https://github.com/conda/conda-libmamba-solver/pull/159 - MINIO_RELEASE: 'archive/minio.RELEASE.2023-03-13T19-46-17Z' - jobs: # detect whether any code changes are included in this PR changes: runs-on: ubuntu-latest permissions: + # necessary to detect changes + # https://github.com/dorny/paths-filter#supported-workflows pull-requests: read outputs: code: ${{ steps.filter.outputs.code }} steps: - uses: actions/checkout@v3 # dorny/paths-filter needs git clone for push events - # https://github.com/marketplace/actions/paths-changes-filter#supported-workflows - # CONDA-LIBMAMBA-SOLVER CHANGE + # https://github.com/dorny/paths-filter#supported-workflows if: github.event_name != 'pull_request' - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 + - uses: dorny/paths-filter@v2.11.1 id: filter with: - # CONDA-LIBMAMBA-SOLVER CHANGE - # changed some paths: filters: | code: - 'conda_libmamba_solver/**' @@ -58,7 +52,7 @@ jobs: - '*.py' - 'recipe/**' - '.github/workflows/upstream_tests.yml' - # /CONDA-LIBMAMBA-SOLVER CHANGE + - 'dev/**' # windows test suite windows: @@ -81,12 +75,10 @@ jobs: env: OS: Windows PYTHON: ${{ matrix.python-version }} - CONDA_SUBDIR: ${{ matrix.conda-subdir }} TEST_SPLITS: 3 TEST_GROUP: ${{ matrix.test-group }} steps: - - name: Checkout conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE @@ -122,16 +114,6 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-py${{ env.PYTHON }}-${{ matrix.default-channel }}-${{ hashFiles('recipe/meta.yaml', 'dev/windows/setup.bat', 'tests/requirements.txt') }} - - name: Cache minio - uses: actions/cache@v3 - env: - # Increase this value to reset cache - CACHE_NUMBER: 3 - with: - path: minio.exe - key: - ${{ runner.os }}-${{ env.MINIO_RELEASE && env.MINIO_RELEASE || 'minio' }}-${{ env.CACHE_NUMBER }} - - name: Set temp dirs correctly # https://github.com/actions/virtual-environments/issues/712 run: | @@ -139,13 +121,18 @@ jobs: echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV - - name: Setup environment + - name: Setup environment for defaults shell: cmd - # CONDA-LIBMAMBA-SOLVER CHANGE - working-directory: conda + if: matrix.default-channel == 'defaults' + run: | + call .\dev\windows\setup.bat + + - name: Setup environment for conda-forge + shell: cmd + if: matrix.default-channel == 'conda-forge' env: CONDA_CHANNELS: ${{ matrix.default-channel }} - CONDA_CHANNEL_PRIORITY: ${{ matrix.default-channel == 'conda-forge' && 'strict' || 'flexible' }} + CONDA_CHANNEL_PRIORITY: strict run: | :: extend conda requirements with our own type "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\requirements.txt" >> .\tests\requirements.txt @@ -165,31 +152,27 @@ jobs: if errorlevel 1 exit 1 # /CONDA-LIBMAMBA-SOLVER CHANGE - - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.conda-subdir }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} - working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE + - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} shell: cmd - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE + working-directory: conda run: | call .\dev\windows\${{ matrix.test-type }}.bat - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. - name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.conda-subdir }}-${{ matrix.test-type }}-${{ matrix.test-group }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths + name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} path: | - conda/.coverage - conda/.test_durations_${OS} - conda/test-report.xml + .coverage + .\tools\durations\${{ runner.os }}.json + test-report.xml retention-days: 1 # linux test suite @@ -236,108 +219,29 @@ jobs: # /CONDA-LIBMAMBA-SOLVER CHANGE - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE; - # we also added '-e CONDA_SOLVER' to the docker options below - # changes the paths to the volume(s) (plural, we also need conda-libmamba-solver) - # and changed the script being run to our vendored copy (last line) run: > - docker run --rm - -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src - -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src + docker run + --rm -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src + --rm -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src -e TEST_SPLITS -e TEST_GROUP - -e CONDA_SOLVER ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }}${{ matrix.default-channel == 'conda-forge' && '-conda-forge' || '' }} /opt/conda-libmamba-solver-src/dev/linux/upstream_${{ matrix.test-type }}.sh - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths - path: | - conda/.coverage - conda/.test_durations_${OS} - conda/test-report.xml - retention-days: 1 - - # linux-qemu test suite - linux-qemu: - # only run test suite if there are code changes - needs: changes - if: false # needs.changes.outputs.code == 'true' # CONDA-LIBMAMBA-SOLVER CHANGE - - # Run one single fast test per docker+qemu emulated linux platform to test that - # test execution is possible there (container+tools+dependencies work). Can be - # changed / extended to run specific tests in case there are platform related - # things to test. Running more tests is time consuming due to emulation - # (factor 2-10x slower). - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - default-channel: ['defaults', 'conda-forge'] - python-version: ['3.11'] - platform: ['arm64', 'ppc64le'] - env: - OS: linux-${{ matrix.platform }} - PYTHON: ${{ matrix.python-version }} - steps: - - name: Checkout conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE - uses: actions/checkout@v3 - with: - fetch-depth: 0 - repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE - path: conda # CONDA-LIBMAMBA-SOLVER CHANGE - - # CONDA-LIBMAMBA-SOLVER CHANGE - - name: Checkout conda-libmamba-solver - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: conda-libmamba-solver - # /CONDA-LIBMAMBA-SOLVER CHANGE - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - # Equivalent to locally run: - # `docker run --privileged --rm tonistiigi/binfmt --install all` - - # CONDA-LIBMAMBA-SOLVER CHANGE - # - export CONDA_SOLVER - # - - - name: Python linux-${{ matrix.platform }} on ${{ matrix.default-channel }}, ${{ matrix.python-version }} tests - run: > - docker run - --rm - -v ${PWD}:/opt/conda-src - -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src - --platform linux/${{ matrix.platform }} - -e TEST_SPLITS - -e TEST_GROUP - ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }}${{ matrix.default-channel == 'conda-forge' && '-conda-forge' || '' }} - bash -c "source /opt/conda/etc/profile.d/conda.sh; \ - pytest --cov=conda -k test_DepsModifier_contract" - - - name: Upload test results - if: always() - uses: actions/upload-artifact@v3 - with: - # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate - # when locally dowloading and comparing results of different workflow runs. - name: test-results-${{ github.sha }}-linux-${{ matrix.platform }}-qemu-${{ matrix.default-channel }}-${{ matrix.python-version }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | conda/.coverage + conda/./tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 @@ -401,16 +305,6 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-py${{ env.PYTHON }}-${{ matrix.default-channel }}-${{ hashFiles('recipe/meta.yaml', 'dev/macos/setup.sh', 'tests/requirements.txt') }} - - name: Cache minio - uses: actions/cache@v3 - env: - # Increase this value to reset cache - CACHE_NUMBER: 3 - with: - path: minio - key: - ${{ runner.os }}-${{ env.MINIO_RELEASE && env.MINIO_RELEASE || 'minio' }}-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@v2 name: Setup miniconda for defaults if: matrix.default-channel == 'defaults' @@ -449,35 +343,33 @@ jobs: - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} shell: bash -el {0} working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE - env: - CONDA_SOLVER: libmamba # CONDA-LIBMAMBA-SOLVER CHANGE run: | ./dev/macos/${{ matrix.test-type }}.sh - uses: codecov/codecov-action@v3 with: - flags: ${{ matrix.test-type }} - env_vars: OS,PYTHON + flags: ${{ matrix.test-type }},${{ runner.os }},${{ matrix.python-version }} - name: Upload test results - if: always() + if: '!cancelled()' uses: actions/upload-artifact@v3 with: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} - # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | conda/.coverage - conda/.test_durations_${OS} + conda/tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 # aggregate and upload aggregate: # only aggregate test suite if there are code changes - needs: [changes, windows, linux, linux-qemu, macos] - if: needs.changes.outputs.code == 'true' || github.event_name == 'schedule' + needs: [changes, windows, linux, macos] + if: >- + !cancelled() + && needs.changes.outputs.code == 'true' runs-on: ubuntu-latest steps: @@ -501,8 +393,8 @@ jobs: # required check analyze: name: Analyze results - needs: [windows, linux, linux-qemu, macos, aggregate] - if: always() + needs: [windows, linux, macos, aggregate] + if: '!cancelled()' runs-on: ubuntu-latest steps: @@ -537,15 +429,14 @@ jobs: # only build canary build if # - prior steps succeeded, # - this is the main repo, and - # - event triggered by push, and # CONDA-LIBMAMBA-SOLVER CHANGE - # - we are on the main (or feature) branch + # - we are on the main, feature, or release branch if: >- - success() + !cancelled() && !github.event.repository.fork - && github.event_name == 'push' && ( github.ref_name == 'main' || startsWith(github.ref_name, 'feature/') + || endsWith(github.ref_name, '.x') ) strategy: matrix: @@ -566,8 +457,34 @@ jobs: clean: true fetch-depth: 0 + # Explicitly use Python 3.11 since each of the OSes has a different default Python + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Detect label + shell: python + run: | + from pathlib import Path + from re import match + from os import environ + + if "${{ github.ref_name }}" == "main": + # main branch commits are uploaded to the dev label + label = "dev" + elif "${{ github.ref_name }}".startswith("feature/"): + # feature branch commits are uploaded to a custom label + label = "${{ github.ref_name }}" + else: + # release branch commits are added to the rc label + # see https://github.com/conda/infrastructure/issues/760 + _, name = "${{ github.repository }}".split("/") + label = f"rc-{name}-${{ github.ref_name }}" + + Path(environ["GITHUB_ENV"]).write_text(f"ANACONDA_ORG_LABEL={label}") + - name: Create and upload canary build - uses: conda/actions/canary-release@v22.10.0 + uses: conda/actions/canary-release@v23.7.0 env: # Run conda-build in isolated activation to properly package conda _CONDA_BUILD_ISOLATED_ACTIVATION: 1 @@ -575,5 +492,5 @@ jobs: package-name: ${{ github.event.repository.name }} subdir: ${{ matrix.subdir }} anaconda-org-channel: conda-canary - anaconda-org-label: ${{ github.ref_name == 'main' && 'dev' || github.ref_name }} + anaconda-org-label: ${{ env.ANACONDA_ORG_LABEL }} anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }} From 2201a9064117ab6a667ae53992d314f0e8112910 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 10 Nov 2023 12:33:45 +0100 Subject: [PATCH 3/4] fix workflow --- .github/workflows/upstream_tests.yml | 121 ++++++--------------------- 1 file changed, 26 insertions(+), 95 deletions(-) diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 4a73f1ef..3804612b 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -14,9 +14,12 @@ on: # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request pull_request: - # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch workflow_dispatch: + # CONDA-LIBMAMBA-SOLVER CHANGE + schedule: + - cron: "15 7 * * 1-5" # Mon to Fri, 7:15am + # /CONDA-LIBMAMBA-SOLVER CHANGE concurrency: # Concurrency group that uses the workflow name and PR number if available @@ -45,6 +48,8 @@ jobs: - uses: dorny/paths-filter@v2.11.1 id: filter with: + # CONDA-LIBMAMBA-SOLVER CHANGE + # changed some paths: filters: | code: - 'conda_libmamba_solver/**' @@ -53,6 +58,7 @@ jobs: - 'recipe/**' - '.github/workflows/upstream_tests.yml' - 'dev/**' + # /CONDA-LIBMAMBA-SOLVER CHANGE # windows test suite windows: @@ -78,7 +84,8 @@ jobs: TEST_SPLITS: 3 TEST_GROUP: ${{ matrix.test-group }} steps: - - uses: actions/checkout@v3 + - name: Checkout conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE + uses: actions/checkout@v3 with: fetch-depth: 0 repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE @@ -121,18 +128,13 @@ jobs: echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV - - name: Setup environment for defaults - shell: cmd - if: matrix.default-channel == 'defaults' - run: | - call .\dev\windows\setup.bat - - - name: Setup environment for conda-forge + - name: Setup environment shell: cmd - if: matrix.default-channel == 'conda-forge' + # CONDA-LIBMAMBA-SOLVER CHANGE + working-directory: conda env: CONDA_CHANNELS: ${{ matrix.default-channel }} - CONDA_CHANNEL_PRIORITY: strict + CONDA_CHANNEL_PRIORITY: ${{ matrix.default-channel == 'conda-forge' && 'strict' || 'flexible' }} run: | :: extend conda requirements with our own type "%GITHUB_WORKSPACE%\conda-libmamba-solver\dev\requirements.txt" >> .\tests\requirements.txt @@ -153,8 +155,8 @@ jobs: # /CONDA-LIBMAMBA-SOLVER CHANGE - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} + working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE shell: cmd - working-directory: conda run: | call .\dev\windows\${{ matrix.test-type }}.bat @@ -169,10 +171,11 @@ jobs: # name has to be unique, to not overwrite uploads of other matrix runs. sha1 is optional and only to differentiate # when locally dowloading and comparing results of different workflow runs. name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} + # CONDA-LIBMAMBA-SOLVER CHANGE: need to prepend conda/ to the paths path: | - .coverage - .\tools\durations\${{ runner.os }}.json - test-report.xml + conda/.coverage + conda/tools/durations/${{ runner.os }}.json + conda/test-report.xml retention-days: 1 # linux test suite @@ -219,10 +222,13 @@ jobs: # /CONDA-LIBMAMBA-SOLVER CHANGE - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} + # CONDA-LIBMAMBA-SOLVER CHANGES: + # - changed the paths to the volume(s) (plural, we also need conda-libmamba-solver) + # - changed the script being run to our vendored copy (last line) run: > - docker run - --rm -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src - --rm -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src + docker run --rm + -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src + -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src -e TEST_SPLITS -e TEST_GROUP ghcr.io/conda/conda-ci:main-linux-python${{ matrix.python-version }}${{ matrix.default-channel == 'conda-forge' && '-conda-forge' || '' }} @@ -241,7 +247,7 @@ jobs: name: test-results-${{ github.sha }}-${{ runner.os }}-${{ matrix.default-channel }}-${{ matrix.python-version }}-${{ matrix.test-type }}-${{ matrix.test-group }} path: | conda/.coverage - conda/./tools/durations/${{ runner.os }}.json + conda/tools/durations/${{ runner.os }}.json conda/test-report.xml retention-days: 1 @@ -367,9 +373,7 @@ jobs: aggregate: # only aggregate test suite if there are code changes needs: [changes, windows, linux, macos] - if: >- - !cancelled() - && needs.changes.outputs.code == 'true' + if: (!cancelled() && needs.changes.outputs.code == 'true') || github.event_name == 'schedule' runs-on: ubuntu-latest steps: @@ -421,76 +425,3 @@ jobs: filename: .github/TEST_FAILURE_REPORT_TEMPLATE.md update_existing: true # /CONDA-LIBMAMBA-SOLVER CHANGE - - # canary builds - build: - name: Canary Build - needs: [analyze] - # only build canary build if - # - prior steps succeeded, - # - this is the main repo, and - # - we are on the main, feature, or release branch - if: >- - !cancelled() - && !github.event.repository.fork - && ( - github.ref_name == 'main' - || startsWith(github.ref_name, 'feature/') - || endsWith(github.ref_name, '.x') - ) - strategy: - matrix: - include: - - runner: ubuntu-latest - subdir: linux-64 - - runner: macos-latest - subdir: osx-64 - - runner: windows-latest - subdir: win-64 - runs-on: ${{ matrix.runner }} - steps: - # Clean checkout of specific git ref needed for package metadata version - # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR: - - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - clean: true - fetch-depth: 0 - - # Explicitly use Python 3.11 since each of the OSes has a different default Python - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Detect label - shell: python - run: | - from pathlib import Path - from re import match - from os import environ - - if "${{ github.ref_name }}" == "main": - # main branch commits are uploaded to the dev label - label = "dev" - elif "${{ github.ref_name }}".startswith("feature/"): - # feature branch commits are uploaded to a custom label - label = "${{ github.ref_name }}" - else: - # release branch commits are added to the rc label - # see https://github.com/conda/infrastructure/issues/760 - _, name = "${{ github.repository }}".split("/") - label = f"rc-{name}-${{ github.ref_name }}" - - Path(environ["GITHUB_ENV"]).write_text(f"ANACONDA_ORG_LABEL={label}") - - - name: Create and upload canary build - uses: conda/actions/canary-release@v23.7.0 - env: - # Run conda-build in isolated activation to properly package conda - _CONDA_BUILD_ISOLATED_ACTIVATION: 1 - with: - package-name: ${{ github.event.repository.name }} - subdir: ${{ matrix.subdir }} - anaconda-org-channel: conda-canary - anaconda-org-label: ${{ env.ANACONDA_ORG_LABEL }} - anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }} From 262df0baf0c926fd5d209ff22dcd133285d0f0c8 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 10 Nov 2023 12:35:52 +0100 Subject: [PATCH 4/4] pre-commit --- .github/workflows/upstream_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 3804612b..c4f84b01 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -224,9 +224,9 @@ jobs: - name: Python ${{ matrix.python-version }} on ${{ matrix.default-channel }}, ${{ matrix.test-type }} tests, group ${{ matrix.test-group }} # CONDA-LIBMAMBA-SOLVER CHANGES: # - changed the paths to the volume(s) (plural, we also need conda-libmamba-solver) - # - changed the script being run to our vendored copy (last line) + # - changed the script being run to our vendored copy (last line) run: > - docker run --rm + docker run --rm -v ${GITHUB_WORKSPACE}/conda:/opt/conda-src -v ${GITHUB_WORKSPACE}/conda-libmamba-solver:/opt/conda-libmamba-solver-src -e TEST_SPLITS