Fix a few things for external scenarios.

premise/__init__.py

@@ -4,7 +4,7 @@
     "clear_inventory_cache",
     "get_regions_definition",
 )
-__version__ = (2, 1, 1, "dev1")
+__version__ = (2, 1, 1, "dev2")
 
 
 from premise.new_database import NewDatabase

premise/external.py

@@ -14,6 +14,7 @@
 import yaml
 from datapackage import Package
 from wurst import searching as ws
+from functools import lru_cache
 
 from .activity_maps import InventorySet
 from .clean_datasets import get_biosphere_flow_uuid
@@ -431,7 +432,7 @@ def __init__(
 
         external_scenario_regions = []
         for datapackage_number, datapackage in enumerate(self.datapackages):
-            external_scenario_regions.append(
+            external_scenario_regions.extend(
                 self.external_scenarios_data[datapackage_number]["regions"]
             )
 
@@ -451,15 +452,22 @@ def regionalize_inventories(self, ds_names, regions, data: dict) -> None:
 
         """
 
+        processed = []
+
         for ds in ws.get_many(
             self.database,
             ws.equals("regionalize", True),
-            ws.either(*[ws.contains("name", name) for name in ds_names]),
+            ws.either(*[ws.contains("name", name) for name in list(set(ds_names))]),
         ):
+            # remove "regionalize" tag
+            if "regionalize" in ds:
+                del ds["regionalize"]
+
+            if ds["location"] not in regions and ds["name"] not in processed:
+                processed.append(ds["name"])
 
-            # Check if datasets already exist for IAM regions
-            # if not, create them
-            if ds["location"] not in regions:
+                # Check if datasets already exist for IAM regions
+                # if not, create them
                 new_acts = self.fetch_proxies(
                     name=ds["name"],
                     ref_prod=ds["reference product"],
@@ -506,9 +514,7 @@ def regionalize_inventories(self, ds_names, regions, data: dict) -> None:
                     self.write_log(act)
                     self.add_to_index(act)
 
-            # remove "regionalize" tag
-            if "regionalize" in ds:
-                del ds["regionalize"]
+
 
         # some datasets might be meant to replace the supply
         # of other datasets, so we need to adjust those
@@ -904,19 +910,21 @@ def write_suppliers_exchanges(self, suppliers: dict, supply_share: float) -> lis
 
         return new_excs
 
+    @lru_cache()
     def add_additional_exchanges(
-        self, additional_exc: dict, region: str, ei_version: str
+            self,
+            name: str,
+            ref_prod: str,
+            categories: tuple,
+            unit: str,
+            amount: float,
+            region: str,
+            ei_version: str
     ) -> list:
         """
         Add additional exchanges to a dataset.
         """
 
-        name = additional_exc["name"]
-        ref_prod = additional_exc.get("reference product")
-        categories = additional_exc.get("categories")
-        unit = additional_exc.get("unit")
-        amount = additional_exc["amount"]
-
         # we need to ensure that the dataset exists
         # to do so, we need to load migration.csv
         # and check if the dataset is there
@@ -928,7 +936,7 @@ def add_additional_exchanges(
             name = mapping[(name, ref_prod)]["name"]
             ref_prod = mapping[(name, ref_prod)]["reference product"]
 
-        if ref_prod:
+        if ref_prod is not None:
             # this is a technosphere exchange
             if region in self.geo.iam_regions:
                 ecoinvent_regions = self.geo.iam_to_ecoinvent_location(region)
@@ -956,7 +964,8 @@ def add_additional_exchanges(
             return self.write_suppliers_exchanges(suppliers, amount)
 
         # this is a biosphere exchange
-        categories = tuple(categories.split("::"))
+        if not isinstance(categories, tuple):
+            categories = tuple(categories.split("::"))
         if len(categories) == 1:
             key = (name, categories[0], "unspecified", unit)
         else:
@@ -1234,9 +1243,13 @@ def create_markets(self) -> None:
                             if "add" in market_vars:
                                 for additional_exc in market_vars["add"]:
                                     add_excs = self.add_additional_exchanges(
-                                        additional_exc,
-                                        region,
-                                        dp.descriptor["ecoinvent"]["version"],
+                                        name=additional_exc["name"],
+                                        ref_prod=additional_exc.get("reference product"),
+                                        categories=additional_exc.get("categories"),
+                                        unit=additional_exc.get("unit"),
+                                        amount=additional_exc.get("amount"),
+                                        region=region,
+                                        ei_version=dp.descriptor["ecoinvent"]["version"],
                                     )
                                     new_market["exchanges"].extend(add_excs)
 

premise/external_data_validation.py

@@ -54,7 +54,7 @@ def find_iam_efficiency_change(
 
 
 def flag_activities_to_adjust(
-    dataset: dict, scenario_data: dict, year: int, dataset_vars: dict, region=None
+    dataset: dict, scenario_data: dict, year: int, dataset_vars: dict, region_proxy_mapping=None
 ) -> dict:
     """
     Flag datasets that will need to be adjusted.
@@ -186,11 +186,8 @@ def flag_activities_to_adjust(
     if dataset_vars["regionalize"]:
         dataset["regionalize"] = dataset_vars["regionalize"]
 
-    if region is not None:
-        if "region mapping" not in dataset:
-            dataset["region mapping"] = {region: dataset["location"]}
-        else:
-            dataset["region mapping"].update({region: dataset["location"]})
+    if region_proxy_mapping is not None:
+        dataset["region mapping"] = region_proxy_mapping
 
     if "production volume variable" in dataset_vars:
         dataset["production volume variable"] = dataset_vars[
@@ -489,7 +486,7 @@ def adjust_candidates_or_raise_error(
         year,
         val,
         inventory_data,
-    ) -> None | list:
+    ) -> [None, list]:
         """Adjust candidates if possible or raise an error if no valid candidates are found."""
         if not candidates:
             if not find_candidates_by_key(inventory_data, key):
@@ -505,7 +502,13 @@ def adjust_candidates_or_raise_error(
             short_listed = short_list_candidates(candidates, scenario_data)
             for region, ds in short_listed.items():
                 if ds is not None:
-                    adjust_candidate(ds, scenario_data, year, val, region)
+                    adjust_candidate(
+                        ds,
+                        scenario_data,
+                        year,
+                        val,
+                        {r: d["location"] for r, d in short_listed.items()},
+                    )
                 else:
                     print(f"No candidate found for {key[0]} and {key[1]} for {region}.")
             return list(short_listed.values())

premise/utils.py

@@ -310,7 +310,6 @@ def hide_messages():
     print("Keep uncertainty data?")
     print("NewDatabase(..., keep_uncertainty_data=True)")
     print("")
-    print("")
     print("Hide these messages?")
     print("NewDatabase(..., quiet=True)")