Merge pull request #191 from PixelgenTechnologies/release-0.18.3

Release 0.18.3

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.3.0"
+    rev: "v0.6.1"
     hooks:
       - id: ruff
       - id: ruff-format

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.18.3] - 2024-09-26
+
+### Fixed
+
+ - Improved memory usage when aggregating PXL files with precomputed layouts.
+
 ## [0.18.2] - 2024-07-16
 
 ### Changed

CITATIONS.md

@@ -4,7 +4,7 @@
 
 - [pixelator](https://doi.org/10.1101/2023.06.05.543770)
 
-  > Karlsson, Filip, Tomasz Kallas, Divya Thiagarajan, Max Karlsson, Maud Schweitzer, Jose Fernandez Navarro, Louise Leijonancker, et al. “Molecular Pixelation: Single Cell Spatial Proteomics by Sequencing.” bioRxiv, June 8, 2023. https://doi.org/10.1101/2023.06.05.543770.
+  > Karlsson, Filip, Tomasz Kallas, Divya Thiagarajan, Max Karlsson, Maud Schweitzer, Jose Fernandez Navarro, Louise Leijonancker, et al. “Molecular pixelation: spatial proteomics of single cells by sequencing.” Nature Methods, May 8, 2024. https://doi.org/10.1038/s41592-024-02268-9.
 
 - [cutadapt](http://dx.doi.org/10.14806/ej.17.1.200)
 

README.md

@@ -3,7 +3,7 @@
 
 ![python-version](https://img.shields.io/badge/python-3.10%20%7C%203.11-blue)
 [![MIT](https://img.shields.io/badge/license-MIT-blue)](https://opensource.org/licenses/MIT)
-[![DOI](https://img.shields.io/badge/DOI-10.1101/2023.06.05.543770-B31B1B.svg)](https://www.biorxiv.org/content/10.1101/2023.06.05.543770v1)
+[![DOI](https://img.shields.io/badge/DOI-10.1038/S41592--024--02268--9-B31B1B.svg)](https://doi.org/10.1038/s41592-024-02268-9)
 [![conda](https://anaconda.org/bioconda/pixelator/badges/version.svg)](https://bioconda.github.io/recipes/pixelator/README.html#package-pixelator)
 [![pypi](https://img.shields.io/pypi/v/pixelgen-pixelator)](https://pypi.org/project/pixelgen-pixelator/)
 [![Tests](https://github.com/PixelgenTechnologies/pixelator/actions/workflows/tests.yml/badge.svg)](https://github.com/PixelgenTechnologies/pixelator/actions/workflows/tests.yml)
@@ -113,7 +113,7 @@ Pixelator is developed and maintained by the [developers](https://github.com/Pix
 
 When using pixelator in your research, please cite the following publication:
 
-> Karlsson, Filip, Tomasz Kallas, Divya Thiagarajan, Max Karlsson, Maud Schweitzer, Jose Fernandez Navarro, Louise Leijonancker, _et al._ “Molecular Pixelation: Single Cell Spatial Proteomics by Sequencing.” bioRxiv, June 8, 2023. https://doi.org/10.1101/2023.06.05.543770.
+> Karlsson, Filip, Tomasz Kallas, Divya Thiagarajan, Max Karlsson, Maud Schweitzer, Jose Fernandez Navarro, Louise Leijonancker, _et al._ "Molecular pixelation: spatial proteomics of single cells by sequencing." Nature Methods, May 8, 2024. https://doi.org/10.1038/s41592-024-02268-9.
 
 
 Main development happened thanks to:

containers/base.Dockerfile

@@ -4,7 +4,7 @@ ARG USE_ENTRYPOINT=false
 ARG MAKEJOBS=4
 
 # Install pixelator dependencies in a separate stage to improve caching
-FROM registry.fedoraproject.org/fedora-minimal:39 as runtime-base
+FROM registry.fedoraproject.org/fedora-minimal:40 as runtime-base
 RUN microdnf install -y \
         python3.11 \
         git \
@@ -49,7 +49,7 @@ RUN microdnf install -y \
      && microdnf clean all
 
 
-# Build Fastp from source
+# Build Fastp and isal from source
 FROM builder-base as build-fastp
 
 RUN git clone https://github.com/intel/isa-l.git
@@ -86,10 +86,10 @@ ENV ANNOY_TARGET_VARIANT="${TARGETVARIANT:-v3}"
 RUN if [ -n "$ANNOY_TARGET_VARIANT" ]; then \
     export ANNOY_COMPILER_ARGS="-D_CRT_SECURE_NO_WARNINGS,-DANNOYLIB_MULTITHREADED_BUILD,-march=x86-64-$ANNOY_TARGET_VARIANT"; \
     echo "Building Annoy for explicit target $TARGETPLATFORM/$ANNOY_TARGET_VARIANT"; \
-    pip3.11 install --prefix=/runtime -r requirements.txt; \
+    pip3.11 install -I --prefix=/runtime -r requirements.txt; \
    else \
         echo "Building Annoy without implicit target $TARGETPLATFORM"; \
-        pip3.11 install --prefix=/runtime -r requirements.txt; \
+        pip3.11 install -I --prefix=/runtime -r requirements.txt; \
     fi \
     && rm requirements.txt
 
@@ -101,7 +101,7 @@ COPY poetry.lock pyproject.toml /pixelator/
 COPY .git /pixelator/.git
 
 RUN poetry export --output requirements.txt --without-hashes --no-interaction --no-ansi
-RUN pip3.11 install --prefix=/runtime -r requirements.txt && rm requirements.txt
+RUN pip3.11 install -I --prefix=/runtime -r requirements.txt && rm requirements.txt
 
 # ------------------------------------------
 # -- Build the pixelator package
@@ -132,7 +132,7 @@ FROM runtime-base as runtime-amd64
 
 # Copy both fastp executable and isa-l library
 COPY --from=build-fastp /usr/local/ /usr/local/
-COPY --from=poetry-deps-install-amd64 /runtime/ /usr/local/
+COPY --from=poetry-deps-install-amd64 /runtime/ /usr/
 
 # ------------------------------------------
 # -- Build the runtime environment for arm64
@@ -142,7 +142,7 @@ FROM runtime-base as runtime-arm64
 
 # Copy both fastp executable and isa-l library
 COPY --from=build-fastp /usr/local/ /usr/local/
-COPY --from=poetry-deps-install-arm64 /runtime/ /usr/local/
+COPY --from=poetry-deps-install-arm64 /runtime/ /usr/
 
 # ------------------------------------------
 # -- Build the final image
@@ -154,12 +154,11 @@ FROM runtime-${TARGETARCH} as runtime-final
 # We add this explicitly since nextflow often runs with PYTHONNOUSERSITE set
 # to fix interference with conda and this can cause problems.
 # Fastp will also build isal and we need to make that available
-ENV PYTHONPATH="$PYTHONPATH:/usr/local/lib/python3.11/site-packages:/usr/local/lib64/python3.11/site-packages"
 RUN ldconfig /usr/local/lib64
 
 COPY --from=build-pixelator /dist /dist
 RUN ls -alh /dist/
-RUN pip3.11 install /dist/*.tar.gz
+RUN pip3.11 install --prefix /usr/ /dist/*.tar.gz
 RUN rm -rf /dist
 
 RUN pip3.11 cache purge

containers/prod.Dockerfile

@@ -11,7 +11,7 @@ LABEL org.opencontainers.image.licenses = "MIT"
 
 # Install pixelator dependencies in a separate stage to improve caching
 FROM runtime AS entrypoint-true
-ENTRYPOINT [ "/usr/local/bin/pixelator" ]
+ENTRYPOINT [ "/usr/bin/pixelator" ]
 
 FROM runtime AS entrypoint-false
 ENTRYPOINT []

src/pixelator/pixeldataset/__init__.py

@@ -210,14 +210,18 @@ def metadata(self, value: Dict) -> None:
     @property
     def precomputed_layouts(
         self,
-    ) -> PreComputedLayouts | None:
+    ) -> PreComputedLayouts:
         """Get the precomputed layouts."""
         return self._backend.precomputed_layouts
 
     @precomputed_layouts.setter
     def precomputed_layouts(self, value: PreComputedLayouts | None) -> None:
         """Set the precomputed layouts."""
-        self._backend.precomputed_layouts = value
+        # Note that the type ignore here is to handle the fact that the setter
+        # needs to be able to take None (in order to make it easier to the user)
+        # but that will be transformed into a empty PreComputedLayouts object
+        # by the backend as and when it sees fit.
+        self._backend.precomputed_layouts = value  # type: ignore
 
     def graph(
         self,

src/pixelator/pixeldataset/aggregation.py

@@ -148,10 +148,7 @@ def _add_sample_name_as_obs_col(adata, name):
     }
 
     precomputed_layouts = aggregate_precomputed_layouts(
-        [
-            (name, dataset.precomputed_layouts)
-            for name, dataset in zip(sample_names, datasets)
-        ],
+        [(name, dataset) for name, dataset in zip(sample_names, datasets)],
         all_markers=set(datasets[0].adata.var.index),
     )
 

src/pixelator/pixeldataset/backends.py

@@ -82,7 +82,7 @@ def metadata(self, value: Dict) -> Optional[Dict]:
         """Set the metadata object."""
 
     @property
-    def precomputed_layouts(self) -> Optional[PreComputedLayouts]:
+    def precomputed_layouts(self) -> PreComputedLayouts:
         """Get the precomputed layouts for the component graphs.
 
         Please note that since these have been pre-computed, if you have made
@@ -91,12 +91,12 @@ def precomputed_layouts(self) -> Optional[PreComputedLayouts]:
         ...
 
     @precomputed_layouts.setter
-    def precomputed_layouts(self, value: PreComputedLayouts) -> None:
+    def precomputed_layouts(self, value: PreComputedLayouts | None) -> None:
         """Set the precomputed layouts for the component graphs."""
         ...
 
 
-class ObjectBasedPixelDatasetBackend:
+class ObjectBasedPixelDatasetBackend(PixelDatasetBackend):
     """A backend for PixelDataset that is backed by in memory objects.
 
     `ObjectBasedPixelDatasetBackend` provides a backend for PixelDatasets that
@@ -209,12 +209,10 @@ def colocalization(self, value: pd.DataFrame) -> None:
         self._colocalization = value
 
     @property
-    def precomputed_layouts(self) -> PreComputedLayouts | None:
+    def precomputed_layouts(self) -> PreComputedLayouts:
         """Get the precomputed layouts."""
         if self._precomputed_layouts is None:
-            return None
-        if self._precomputed_layouts.is_empty:
-            return None
+            return PreComputedLayouts.create_empty()
         return self._precomputed_layouts
 
     @precomputed_layouts.setter
@@ -225,7 +223,7 @@ def precomputed_layouts(self, value: PreComputedLayouts | None) -> None:
         self._precomputed_layouts = value
 
 
-class FileBasedPixelDatasetBackend:
+class FileBasedPixelDatasetBackend(PixelDatasetBackend):
     """A file based backend for PixelDataset.
 
     `FileBasedPixelDatasetBackend` is used to lazily fetch information from
@@ -260,7 +258,7 @@ def edgelist(self) -> pd.DataFrame:
         return self._datastore.read_edgelist()
 
     @property
-    def edgelist_lazy(self) -> Optional[pl.LazyFrame]:
+    def edgelist_lazy(self) -> pl.LazyFrame:
         """Get a lazy frame representation of the edgelist."""
         return self._datastore.read_edgelist_lazy()
 
@@ -280,19 +278,16 @@ def metadata(self) -> Optional[Dict]:
         return self._datastore.read_metadata()
 
     @property
-    def precomputed_layouts(self) -> PreComputedLayouts | None:
+    def precomputed_layouts(self) -> PreComputedLayouts:
         """Get the precomputed layouts."""
         # If it is None it means it is uninitialized, and we should
         # attempt to read it lazily
-        if self._precomputed_layouts is None:
-            return self._datastore.read_precomputed_layouts()
 
-        # It can also be empty, in which case it has been read and
-        # found to be empty. Or it has been initialized to be empty,
-        # which means that it should be cleared.
-        if self._precomputed_layouts.is_empty:
-            return None
+        if isinstance(self._precomputed_layouts, PreComputedLayouts):
+            return self._precomputed_layouts
 
+        if self._precomputed_layouts is None:
+            self._precomputed_layouts = self._datastore.read_precomputed_layouts()
         return self._precomputed_layouts
 
     @precomputed_layouts.setter