Merge branch 'main' of github.com:scitools/iris into fix-map-blocks-c…

…hunktype

.github/dependabot.yml

@@ -8,8 +8,11 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      # Check for updates to GitHub Actions every weekday
-      interval: "daily"
+      # Check later in the week - the upstream dependabot check in `workflows` runs deliberately early in the week.
+      #  Therefore allowing time for the `workflows` update to be merged-and-released first.
+      interval: "weekly"
+      day: "thursday"
+      time: "01:00"
+      timezone: "Europe/London"
     labels:
-      - "New: Pull Request"
       - "Bot"

.github/workflows/ci-manifest.yml

@@ -23,4 +23,4 @@ concurrency:
 jobs:
   manifest:
     name: "check-manifest"
-    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.07.4
+    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.09.1

.github/workflows/ci-tests.yml

@@ -50,7 +50,7 @@ jobs:
             session: "tests"
 
     env:
-      IRIS_TEST_DATA_VERSION: "2.22"
+      IRIS_TEST_DATA_VERSION: "2.25"
       ENV_NAME: "ci-tests"
 
     steps:

.github/workflows/refresh-lockfiles.yml

@@ -14,5 +14,5 @@ on:
 
 jobs:
   refresh_lockfiles:
-    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.07.4
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.09.1
     secrets: inherit

benchmarks/README.md

@@ -118,6 +118,12 @@ repeats _between_ `setup()` calls using the `repeat` attribute.
 `warmup_time = 0` is also advisable since ASV performs independent re-runs to
 estimate run-time, and these will still be subject to the original problem.
 
+### Custom benchmarks
+
+Iris benchmarking implements custom benchmark types, such as a `tracemalloc`
+benchmark to measure memory growth. See [custom_bms/](./custom_bms) for more
+detail.
+
 ### Scaling / non-Scaling Performance Differences
 
 **(We no longer advocate the below for benchmarks run during CI, given the

benchmarks/asv.conf.json

@@ -53,9 +53,12 @@
     "command_comment": [
         "We know that the Nox command takes care of installation in each",
         "environment, and in the case of Iris no specialised uninstall or",
-        "build commands are needed to get it working."
+        "build commands are needed to get it working.",
+
+        "We do however need to install the custom benchmarks for them to be",
+        "usable."
     ],
     "install_command": [],
     "uninstall_command": [],
-    "build_command": []
+    "build_command": ["python {conf_dir}/custom_bms/install.py"]
 }

benchmarks/benchmarks/__init__.py

@@ -37,111 +37,6 @@ def disable_repeat_between_setup(benchmark_object):
     return benchmark_object
 
 
-class TrackAddedMemoryAllocation:
-    """Measures by how much process resident memory grew, during execution.
-
-    Context manager which measures by how much process resident memory grew,
-    during execution of its enclosed code block.
-
-    Obviously limited as to what it actually measures : Relies on the current
-    process not having significant unused (de-allocated) memory when the
-    tested codeblock runs, and only reliable when the code allocates a
-    significant amount of new memory.
-
-    Example:
-        with TrackAddedMemoryAllocation() as mb:
-            initial_call()
-            other_call()
-        result = mb.addedmem_mb()
-
-    Attributes
-    ----------
-    RESULT_MINIMUM_MB : float
-        The smallest result that should ever be returned, in Mb. Results
-        fluctuate from run to run (usually within 1Mb) so if a result is
-        sufficiently small this noise will produce a before-after ratio over
-        AVD's detection threshold and be treated as 'signal'. Results
-        smaller than this value will therefore be returned as equal to this
-        value, ensuring fractionally small noise / no noise at all.
-        Defaults to 1.0
-
-    RESULT_ROUND_DP : int
-        Number of decimal places of rounding on result values (in Mb).
-        Defaults to 1
-
-    """
-
-    RESULT_MINIMUM_MB = 0.2
-    RESULT_ROUND_DP = 1  # I.E. to nearest 0.1 Mb
-
-    def __enter__(self):
-        tracemalloc.start()
-        return self
-
-    def __exit__(self, *_):
-        _, peak_mem_bytes = tracemalloc.get_traced_memory()
-        tracemalloc.stop()
-        # Save peak-memory allocation, scaled from bytes to Mb.
-        self._peak_mb = peak_mem_bytes * (2.0**-20)
-
-    def addedmem_mb(self):
-        """Return measured memory growth, in Mb."""
-        result = self._peak_mb
-        # Small results are too vulnerable to noise being interpreted as signal.
-        result = max(self.RESULT_MINIMUM_MB, result)
-        # Rounding makes results easier to read.
-        result = np.round(result, self.RESULT_ROUND_DP)
-        return result
-
-    @staticmethod
-    def decorator(decorated_func):
-        """Benchmark to track growth in resident memory during execution.
-
-        Intended for use on ASV ``track_`` benchmarks. Applies the
-        :class:`TrackAddedMemoryAllocation` context manager to the benchmark
-        code, sets the benchmark ``unit`` attribute to ``Mb``.
-
-        """
-
-        def _wrapper(*args, **kwargs):
-            assert decorated_func.__name__[:6] == "track_"
-            # Run the decorated benchmark within the added memory context
-            # manager.
-            with TrackAddedMemoryAllocation() as mb:
-                decorated_func(*args, **kwargs)
-            return mb.addedmem_mb()
-
-        decorated_func.unit = "Mb"
-        return _wrapper
-
-    @staticmethod
-    def decorator_repeating(repeats=3):
-        """Benchmark to track growth in resident memory during execution.
-
-        Tracks memory for repeated calls of decorated function.
-
-        Intended for use on ASV ``track_`` benchmarks. Applies the
-        :class:`TrackAddedMemoryAllocation` context manager to the benchmark
-        code, sets the benchmark ``unit`` attribute to ``Mb``.
-
-        """
-
-        def decorator(decorated_func):
-            def _wrapper(*args, **kwargs):
-                assert decorated_func.__name__[:6] == "track_"
-                # Run the decorated benchmark within the added memory context
-                # manager.
-                with TrackAddedMemoryAllocation() as mb:
-                    for _ in range(repeats):
-                        decorated_func(*args, **kwargs)
-                return mb.addedmem_mb()
-
-            decorated_func.unit = "Mb"
-            return _wrapper
-
-        return decorator
-
-
 def on_demand_benchmark(benchmark_object):
     """Disable these benchmark(s) unless ON_DEMAND_BENCHARKS env var is set.
 

benchmarks/benchmarks/cperf/__init__.py

@@ -14,9 +14,6 @@
 
 from iris import load_cube
 
-# TODO: remove uses of PARSE_UGRID_ON_LOAD once UGRID parsing is core behaviour.
-from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD
-
 from ..generate_data import BENCHMARK_DATA
 from ..generate_data.ugrid import make_cubesphere_testfile
 
@@ -92,5 +89,4 @@ def setup(self, file_type, three_d, three_times):
         self.file_type = file_type
 
     def load(self):
-        with PARSE_UGRID_ON_LOAD.context():
-            return load_cube(str(self.file_path))
+        return load_cube(str(self.file_path))

benchmarks/benchmarks/cperf/save.py

@@ -6,7 +6,7 @@
 
 from iris import save
 
-from .. import TrackAddedMemoryAllocation, on_demand_benchmark
+from .. import on_demand_benchmark
 from ..generate_data.ugrid import make_cube_like_2d_cubesphere, make_cube_like_umfield
 from . import _N_CUBESPHERE_UM_EQUIVALENT, _UM_DIMS_YX
 
@@ -36,6 +36,5 @@ def _save_data(self, cube):
     def time_save_data_netcdf(self, data_type):
         self._save_data(self.cube)
 
-    @TrackAddedMemoryAllocation.decorator
-    def track_addedmem_save_data_netcdf(self, data_type):
+    def tracemalloc_save_data_netcdf(self, data_type):
         self._save_data(self.cube)

benchmarks/benchmarks/generate_data/stock.py

@@ -14,7 +14,7 @@
 
 import iris
 from iris import cube
-from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh
+from iris.mesh import load_mesh
 
 from . import BENCHMARK_DATA, REUSE_DATA, load_realised, run_function_elsewhere
 
@@ -90,7 +90,7 @@ def sample_mesh(n_nodes=None, n_faces=None, n_edges=None, lazy_values=False):
     """Sample mesh wrapper for :meth:iris.tests.stock.mesh.sample_mesh`."""
 
     def _external(*args, **kwargs):
-        from iris.experimental.ugrid import save_mesh
+        from iris.mesh import save_mesh
         from iris.tests.stock.mesh import sample_mesh
 
         save_path_ = kwargs.pop("save_path")
@@ -104,21 +104,20 @@ def _external(*args, **kwargs):
     save_path = (BENCHMARK_DATA / f"sample_mesh_{args_hash}").with_suffix(".nc")
     if not REUSE_DATA or not save_path.is_file():
         _ = run_function_elsewhere(_external, *arg_list, save_path=str(save_path))
-    with PARSE_UGRID_ON_LOAD.context():
-        if not lazy_values:
-            # Realise everything.
-            with load_realised():
-                mesh = load_mesh(str(save_path))
-        else:
+    if not lazy_values:
+        # Realise everything.
+        with load_realised():
             mesh = load_mesh(str(save_path))
+    else:
+        mesh = load_mesh(str(save_path))
     return mesh
 
 
 def sample_meshcoord(sample_mesh_kwargs=None, location="face", axis="x"):
     """Sample meshcoord wrapper for :meth:`iris.tests.stock.mesh.sample_meshcoord`.
 
     Parameters deviate from the original as cannot pass a
-    :class:`iris.experimental.ugrid.Mesh to the separate Python instance - must
+    :class:`iris.mesh.Mesh to the separate Python instance - must
     instead generate the Mesh as well.
 
     MeshCoords cannot be saved to file, so the _external method saves the
@@ -127,7 +126,7 @@ def sample_meshcoord(sample_mesh_kwargs=None, location="face", axis="x"):
     """
 
     def _external(sample_mesh_kwargs_, save_path_):
-        from iris.experimental.ugrid import save_mesh
+        from iris.mesh import save_mesh
         from iris.tests.stock.mesh import sample_mesh, sample_meshcoord
 
         if sample_mesh_kwargs_:
@@ -147,9 +146,8 @@ def _external(sample_mesh_kwargs_, save_path_):
             sample_mesh_kwargs_=sample_mesh_kwargs,
             save_path_=str(save_path),
         )
-    with PARSE_UGRID_ON_LOAD.context():
-        with load_realised():
-            source_mesh = load_mesh(str(save_path))
+    with load_realised():
+        source_mesh = load_mesh(str(save_path))
     # Regenerate MeshCoord from its Mesh, which we saved.
     return source_mesh.to_MeshCoord(location=location, axis=axis)
 
@@ -180,7 +178,6 @@ def _external(w_mesh_: str, save_path_: str):
     )
     if not REUSE_DATA or not save_path.is_file():
         _ = run_function_elsewhere(_external, w_mesh_=w_mesh, save_path_=str(save_path))
-    with PARSE_UGRID_ON_LOAD.context():
-        context = nullcontext() if lazy else load_realised()
-        with context:
-            return iris.load_cube(save_path, "air_potential_temperature")
+    context = nullcontext() if lazy else load_realised()
+    with context:
+        return iris.load_cube(save_path, "air_potential_temperature")

benchmarks/benchmarks/generate_data/ugrid.py

@@ -5,7 +5,6 @@
 """Scripts for generating supporting data for UGRID-related benchmarking."""
 
 from iris import load_cube as iris_loadcube
-from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD
 
 from . import BENCHMARK_DATA, REUSE_DATA, load_realised, run_function_elsewhere
 from .stock import (
@@ -85,8 +84,7 @@ def make_cube_like_2d_cubesphere(n_cube: int, with_mesh: bool):
         )
 
     # File now *should* definitely exist: content is simply the desired cube.
-    with PARSE_UGRID_ON_LOAD.context():
-        cube = iris_loadcube(str(filepath))
+    cube = iris_loadcube(str(filepath))
 
     # Ensure correct laziness.
     _ = cube.data
@@ -155,9 +153,8 @@ def _external(xy_dims_, save_path_):
     )
     if not REUSE_DATA or not save_path.is_file():
         _ = run_function_elsewhere(_external, xy_dims, str(save_path))
-    with PARSE_UGRID_ON_LOAD.context():
-        with load_realised():
-            cube = iris_loadcube(str(save_path))
+    with load_realised():
+        cube = iris_loadcube(str(save_path))
 
     return cube
 

benchmarks/benchmarks/load/ugrid.py

@@ -5,8 +5,7 @@
 """Mesh data loading benchmark tests."""
 
 from iris import load_cube as iris_load_cube
-from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD
-from iris.experimental.ugrid import load_mesh as iris_load_mesh
+from iris.mesh import load_mesh as iris_load_mesh
 
 from ..generate_data.stock import create_file__xios_2d_face_half_levels
 
@@ -18,13 +17,11 @@ def synthetic_data(**kwargs):
 
 
 def load_cube(*args, **kwargs):
-    with PARSE_UGRID_ON_LOAD.context():
-        return iris_load_cube(*args, **kwargs)
+    return iris_load_cube(*args, **kwargs)
 
 
 def load_mesh(*args, **kwargs):
-    with PARSE_UGRID_ON_LOAD.context():
-        return iris_load_mesh(*args, **kwargs)
+    return iris_load_mesh(*args, **kwargs)
 
 
 class BasicLoading: