Ensure that channels match expected value in conda-build (#365)

* add reproducer test

* move test

* use canonical channel name when available

* pre-commit

* add news

* only for conda-build

* parametrize recipes

* try with just the string

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix merge artifact

* use the full channel, not just the name

* pre-commit

* detect whether channel belongs to multichannel via url

* use separate CONDA_BLD_PATH dirs

* only the channels test needs to be isolated

* cache this called_from_conda_build call a bit

* make sure we clear the repo before reloading it

* try shorter env path on windows

* use restricted unicode on windows

* pre-commit

* unicode restrictions on this test only

* extend stackvana example

* pre-commit

* retrigger

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

conda_libmamba_solver/index.py

@@ -155,10 +155,12 @@ def reload_local_channels(self):
         Reload a channel that was previously loaded from a local directory.
         """
         for noauth_url, info in self._index.items():
-            if noauth_url.startswith("file://"):
+            if noauth_url.startswith("file://") or info.channel.scheme == "file":
                 url, json_path = self._fetch_channel(info.full_url)
-                new = self._json_path_to_repo_info(url, json_path)
-                self._repos[self._repos.index(info.repo)] = new.repo
+                repo_position = self._repos.index(info.repo)
+                info.repo.clear(True)
+                new = self._json_path_to_repo_info(url, json_path, try_solv=False)
+                self._repos[repo_position] = new.repo
                 self._index[noauth_url] = new
         set_channel_priorities(self._index)
 
@@ -234,20 +236,26 @@ def _fetch_channel(self, url: str) -> Tuple[str, os.PathLike]:
 
         return url, json_path
 
-    def _json_path_to_repo_info(self, url: str, json_path: str) -> Optional[_ChannelRepoInfo]:
+    def _json_path_to_repo_info(
+        self, url: str, json_path: str, try_solv: bool = False
+    ) -> Optional[_ChannelRepoInfo]:
         channel = Channel.from_url(url)
         noauth_url = channel.urls(with_credentials=False, subdirs=(channel.subdir,))[0]
         json_path = Path(json_path)
-        solv_path = json_path.parent / f"{json_path.stem}.solv"
         try:
             json_stat = json_path.stat()
         except OSError as exc:
             log.debug("Failed to stat %s", json_path, exc_info=exc)
             json_stat = None
-        try:
-            solv_stat = solv_path.stat()
-        except OSError as exc:
-            log.debug("Failed to stat %s", solv_path, exc_info=exc)
+        if try_solv:
+            try:
+                solv_path = json_path.parent / f"{json_path.stem}.solv"
+                solv_stat = solv_path.stat()
+            except OSError as exc:
+                log.debug("Failed to stat %s", solv_path, exc_info=exc)
+                solv_stat = None
+        else:
+            solv_path = None
             solv_stat = None
 
         if solv_stat is None and json_stat is None:

conda_libmamba_solver/solver.py

@@ -26,9 +26,9 @@
     REPODATA_FN,
     UNKNOWN_CHANNEL,
     ChannelPriority,
-    on_win,
 )
 from conda.base.context import context
+from conda.common.compat import on_win
 from conda.common.constants import NULL
 from conda.common.io import Spinner, timeout
 from conda.common.path import paths_equal
@@ -160,7 +160,6 @@ def solve_final_state(
         # From now on we _do_ require a solver and the index
         init_api_context()
         subdirs = self.subdirs
-        conda_bld_channels = ()
         if self._called_from_conda_build():
             log.info("Using solver via 'conda.plan.install_actions' (probably conda build)")
             # Problem: Conda build generates a custom index which happens to "forget" about
@@ -179,6 +178,7 @@ def solve_final_state(
             IndexHelper = _CachedLibMambaIndexHelper
         else:
             IndexHelper = LibMambaIndexHelper
+            conda_bld_channels = ()
 
         all_channels = [
             *conda_bld_channels,
@@ -826,8 +826,11 @@ def _export_solved_records(
             else:
                 log.warn("Tried to unlink %s but it is not installed or manageable?", filename)
 
+        for_conda_build = self._called_from_conda_build()
         for channel, filename, json_payload in to_link:
-            record = self._package_record_from_json_payload(index, channel, filename, json_payload)
+            record = self._package_record_from_json_payload(
+                index, channel, filename, json_payload, for_conda_build=for_conda_build
+            )
             # We need this check below to make sure noarch package get reinstalled
             # record metadata coming from libmamba is incomplete and won't pass the
             # noarch checks -- to fix it, we swap the metadata-only record with its locally
@@ -848,20 +851,28 @@ def _export_solved_records(
             )
 
         # Fixes conda-build tests/test_api_build.py::test_croot_with_spaces
-        if on_win and self._called_from_conda_build():
+        if on_win and for_conda_build:
             for record in out_state.records.values():
-                record.channel.location = percent_decode(record.channel.location)
+                if "%" not in str(record):
+                    continue
+                if record.channel.location:  # multichannels like 'defaults' have no location
+                    record.channel.location = percent_decode(record.channel.location)
                 record.channel.name = percent_decode(record.channel.name)
 
     def _package_record_from_json_payload(
-        self, index: LibMambaIndexHelper, channel: str, pkg_filename: str, json_payload: str
+        self,
+        index: LibMambaIndexHelper,
+        channel: str,
+        pkg_filename: str,
+        json_payload: str,
+        for_conda_build: bool = False,
     ) -> PackageRecord:
         """
         The libmamba transactions cannot return full-blown objects from the C/C++ side.
         Instead, it returns the instructions to build one on the Python side:
 
         channel_info: dict
-            Channel data, as built in .index.LibmambaIndexHelper._fetch_channel()
+            Channel datas, as built in .index.LibmambaIndexHelper._fetch_channel()
             This is retrieved from the .index._index mapping, keyed by channel URLs
         pkg_filename: str
             The filename (.tar.bz2 or .conda) of the selected record.
@@ -887,6 +898,14 @@ def _package_record_from_json_payload(
         # Otherwise, these are records from the index
         kwargs["fn"] = pkg_filename
         kwargs["channel"] = channel_info.channel
+        if for_conda_build:
+            # conda-build expects multichannel instances in the Dist->PackageRecord mapping
+            # see https://github.com/conda/conda-libmamba-solver/issues/363
+            for multichannel_name, mc_channels in context.custom_multichannels.items():
+                urls = [url for c in mc_channels for url in c.urls(with_credentials=False)]
+                if channel_info.noauth_url in urls:
+                    kwargs["channel"] = multichannel_name
+                    break
         kwargs["url"] = join_url(channel_info.full_url, pkg_filename)
         if not kwargs.get("subdir"):  # missing in old channels
             kwargs["subdir"] = channel_info.channel.subdir

news/365-canonical-channel-names

@@ -0,0 +1,19 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* Use canonical channel names (if available) in exported `PackageRecord` objects. Fixes an issue with conda-build and custom multichannels. (#363 via #365)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>

tests/data/conda_build_recipes/stackvana/meta.yaml

@@ -1,5 +1,6 @@
 {% set name = "stackvana-core" %}
 {% set version = "0.2021.43" %}
+{% set eups_product = "lsst_distrib" %}
 
 package:
   name: {{ name|lower }}
@@ -15,11 +16,45 @@ outputs:
       script:
         - echo "BUILDING IMPL" >> $PREFIX/stackvana-core-impl  # [unix]
         - echo "BUILDING IMPL" >> %PREFIX%/stackvana-core-impl  # [win]
+    test:
+      commands:
+        - echo OK
   - name: stackvana-core
     version: {{ version }}
-    run_exports:
-      - {{ pin_subpackage('stackvana-core-impl', exact=True) }}
-
+    build:
+      script:
+        - echo "BUILDING CORE" >> $PREFIX/stackvana-core  # [unix]
+        - echo "BUILDING CORE" >> %PREFIX%/stackvana-core  # [win]
+      run_exports:
+        - {{ pin_subpackage('stackvana-core-impl', exact=True) }}
     requirements:
       run:
         - {{ pin_subpackage('stackvana-core-impl', exact=True) }}
+    test:
+      commands:
+        - echo OK
+  - name: stackvana-{{ eups_product }}
+    version: {{ version }}
+    build:
+      script:
+        - echo "BUILDING {{ eups_product }}" >> $PREFIX/stackvana-{{ eups_product }}  # [unix]
+        - echo "BUILDING {{ eups_product }}" >> %PREFIX%/stackvana-{{ eups_product }}  # [win]
+    requirements:
+      host:
+        - stackvana-core =={{ version }}
+      run:
+        - stackvana-core =={{ version }}
+    test:
+      commands:
+        - echo OK
+  - name: stackvana
+    version: {{ version }}
+    build:
+      script:
+        - echo "BUILDING STACKVANA" >> $PREFIX/stackvana  # [unix]
+        - echo "BUILDING STACKVANA" >> %PREFIX%/stackvana  # [win]
+    requirements:
+        - {{ pin_subpackage("stackvana-" ~ eups_product, max_pin="x.x.x") }}
+    test:
+      commands:
+        - echo OK

tests/test_channels.py

@@ -9,7 +9,7 @@
 
 import pytest
 from conda.base.context import reset_context
-from conda.common.compat import on_linux
+from conda.common.compat import on_linux, on_win
 from conda.common.io import env_vars
 from conda.core.prefix_data import PrefixData
 from conda.models.channel import Channel
@@ -23,6 +23,8 @@
 from .channel_testing.helpers import create_with_channel
 from .utils import conda_subprocess, write_env_config
 
+DATA = Path(__file__).parent / "data"
+
 
 def test_channel_matchspec():
     stdout, *_ = conda_inprocess(
@@ -89,9 +91,19 @@ def test_channels_installed_unavailable():
         assert retcode == 0
 
 
-def _setup_channels_alias(prefix):
+def _setup_conda_forge_as_defaults(prefix, force=False):
+    write_env_config(
+        prefix,
+        force=force,
+        channels=["defaults"],
+        default_channels=["conda-forge"],
+    )
+
+
+def _setup_channels_alias(prefix, force=False):
     write_env_config(
         prefix,
+        force=force,
         channels=["conda-forge", "defaults"],
         channel_alias="https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud",
         migrated_channel_aliases=["https://conda.anaconda.org"],
@@ -103,9 +115,10 @@ def _setup_channels_alias(prefix):
     )
 
 
-def _setup_channels_custom(prefix):
+def _setup_channels_custom(prefix, force=False):
     write_env_config(
         prefix,
+        force=force,
         channels=["conda-forge", "defaults"],
         custom_channels={
             "conda-forge": "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud",
@@ -219,6 +232,32 @@ def test_encoding_file_paths(tmp_path: Path):
     assert list((tmp_path / "env" / "conda-meta").glob("test-package-*.json"))
 
 
+def test_conda_build_with_aliased_channels(tmp_path):
+    "https://github.com/conda/conda-libmamba-solver/issues/363"
+    condarc = Path.home() / ".condarc"
+    condarc_contents = condarc.read_text() if condarc.is_file() else None
+    env = os.environ.copy()
+    if on_win:
+        env["CONDA_BLD_PATH"] = str(Path(os.environ.get("RUNNER_TEMP", tmp_path), "bld"))
+    else:
+        env["CONDA_BLD_PATH"] = str(tmp_path / "conda-bld")
+    try:
+        _setup_conda_forge_as_defaults(Path.home(), force=True)
+        conda_subprocess(
+            "build",
+            DATA / "conda_build_recipes" / "jedi",
+            "--override-channels",
+            "--channel=defaults",
+            capture_output=False,
+            env=env,
+        )
+    finally:
+        if condarc_contents:
+            condarc.write_text(condarc_contents)
+        else:
+            condarc.unlink()
+
+
 def test_http_server_auth_none(http_server_auth_none):
     create_with_channel(http_server_auth_none)
 
@@ -247,12 +286,12 @@ def test_http_server_auth_token_in_defaults(http_server_auth_token):
         )
         reset_context()
         conda_subprocess("info", capture_output=False)
-        conda_inprocess(
+        conda_subprocess(
             "create",
-            _get_temp_prefix(),
+            "-p",
+            _get_temp_prefix(use_restricted_unicode=on_win),
             "--solver=libmamba",
             "test-package",
-            no_capture=True,
         )
     finally:
         if condarc_contents:

tests/test_downstream.py

@@ -11,27 +11,30 @@
 DATA = Path(__file__).parent / "data"
 
 
-def test_build_recipes():
+@pytest.mark.parametrize(
+    "recipe",
+    [
+        pytest.param(x, id=x.name)
+        for x in sorted((DATA / "conda_build_recipes").iterdir())
+        if (x / "meta.yaml").is_file()
+    ],
+)
+def test_build_recipe(recipe):
     """
     Adapted from
     https://github.com/mamba-org/boa/blob/3213180564/tests/test_mambabuild.py#L6
 
     See /tests/data/conda_build_recipes/LICENSE for more details
     """
-    recipes_dir = DATA / "conda_build_recipes"
-
-    recipes = [str(x) for x in recipes_dir.iterdir() if x.is_dir()]
+    expected_fail_recipes = ["baddeps"]
     env = os.environ.copy()
     env["CONDA_SOLVER"] = "libmamba"
-    expected_fail_recipes = ["baddeps"]
-    for recipe in recipes:
-        recipe_name = Path(recipe).name
-        print(f"Running {recipe_name}")
-        if recipe_name in expected_fail_recipes:
-            with pytest.raises(CalledProcessError):
-                check_call(["conda-build", recipe], env=env)
-        else:
+    recipe_name = Path(recipe).name
+    if recipe_name in expected_fail_recipes:
+        with pytest.raises(CalledProcessError):
             check_call(["conda-build", recipe], env=env)
+    else:
+        check_call(["conda-build", recipe], env=env)
 
 
 def test_conda_lock(tmp_path):