From 724c096836fe88b72fd2066f54d7b34570c934d0 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:32:14 +0100 Subject: [PATCH 01/13] don't move to gpu inside prepare_data --- cyto_ml/models/scivision.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cyto_ml/models/scivision.py b/cyto_ml/models/scivision.py index 2aa4de3..efba243 100644 --- a/cyto_ml/models/scivision.py +++ b/cyto_ml/models/scivision.py @@ -34,20 +34,18 @@ def prepare_image(image: DataArray): Take an xarray of image data and prepare it to pass through the model a) Converts the image data to a PyTorch tensor b) Accepts a single image or batch (no need for torch.stack) - c) Uses a CUDA device if available """ + # Computes the DataArray and returns a numpy array + image_numpy = image.to_numpy() + # Convert the image data to a PyTorch tensor - tensor_image = torchvision.transforms.ToTensor()(image.to_numpy()) + tensor_image = torchvision.transforms.ToTensor()(image_numpy) # Check if the input is a single image or a batch if len(tensor_image.shape) == 3: # Single image, add a batch dimension tensor_image = tensor_image.unsqueeze(0) - # Check if CUDA is available and move the tensor to the CUDA device - if torch.cuda.is_available(): - tensor_image = tensor_image.cuda() - return tensor_image From 65cc62a336e49f0804ff0d0ba62631fe0dd5524e Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:37:34 +0100 Subject: [PATCH 02/13] switch to src layout --- .gitignore | 1 + pyproject.toml | 11 ++++++----- {cyto_ml => src/cyto_ml}/__init__.py | 0 {cyto_ml => src/cyto_ml}/data/__init__.py | 0 {cyto_ml => src/cyto_ml}/data/intake.py | 0 {cyto_ml => src/cyto_ml}/data/s3.py | 0 {cyto_ml => src/cyto_ml}/data/vectorstore.py | 0 {cyto_ml => src/cyto_ml}/models/__init__.py | 0 {cyto_ml => src/cyto_ml}/models/scivision.py | 0 {cyto_ml => tests}/tests/conftest.py | 0 .../fixtures/test_images/testymctestface_1091.tif | Bin .../fixtures/test_images/testymctestface_113.tif | Bin .../fixtures/test_images/testymctestface_127.tif | Bin .../fixtures/test_images/testymctestface_133.tif | Bin .../fixtures/test_images/testymctestface_1388.tif | Bin .../fixtures/test_images/testymctestface_1407.tif | Bin .../fixtures/test_images/testymctestface_1830.tif | Bin .../fixtures/test_images/testymctestface_1876.tif | Bin .../fixtures/test_images/testymctestface_188.tif | Bin .../fixtures/test_images/testymctestface_1887.tif | Bin .../fixtures/test_images/testymctestface_1890.tif | Bin .../fixtures/test_images/testymctestface_1892.tif | Bin .../fixtures/test_images/testymctestface_1901.tif | Bin .../fixtures/test_images/testymctestface_1909.tif | Bin .../fixtures/test_images/testymctestface_1912.tif | Bin .../fixtures/test_images/testymctestface_1914.tif | Bin .../fixtures/test_images/testymctestface_1915.tif | Bin .../fixtures/test_images/testymctestface_1919.tif | Bin .../fixtures/test_images/testymctestface_1922.tif | Bin .../fixtures/test_images/testymctestface_1924.tif | Bin .../fixtures/test_images/testymctestface_1948.tif | Bin .../fixtures/test_images/testymctestface_1953.tif | Bin .../fixtures/test_images/testymctestface_1962.tif | Bin .../fixtures/test_images/testymctestface_1965.tif | Bin .../fixtures/test_images/testymctestface_1981.tif | Bin .../fixtures/test_images/testymctestface_2012.tif | Bin .../fixtures/test_images/testymctestface_2071.tif | Bin .../fixtures/test_images/testymctestface_2102.tif | Bin .../fixtures/test_images/testymctestface_2108.tif | Bin .../fixtures/test_images/testymctestface_2110.tif | Bin .../fixtures/test_images/testymctestface_2115.tif | Bin .../fixtures/test_images/testymctestface_2117.tif | Bin .../fixtures/test_images/testymctestface_2119.tif | Bin .../fixtures/test_images/testymctestface_2172.tif | Bin .../fixtures/test_images/testymctestface_2715.tif | Bin .../fixtures/test_images/testymctestface_36.tif | Bin .../fixtures/test_images/testymctestface_3612.tif | Bin .../fixtures/test_images/testymctestface_3814.tif | Bin .../fixtures/test_images/testymctestface_4715.tif | Bin .../fixtures/test_images/testymctestface_4961.tif | Bin {cyto_ml => tests}/tests/test_image_embeddings.py | 0 {cyto_ml => tests}/tests/test_object_store.py | 0 {cyto_ml => tests}/tests/test_prepare_image.py | 0 {cyto_ml => tests}/tests/test_vector_store.py | 0 54 files changed, 7 insertions(+), 5 deletions(-) rename {cyto_ml => src/cyto_ml}/__init__.py (100%) rename {cyto_ml => src/cyto_ml}/data/__init__.py (100%) rename {cyto_ml => src/cyto_ml}/data/intake.py (100%) rename {cyto_ml => src/cyto_ml}/data/s3.py (100%) rename {cyto_ml => src/cyto_ml}/data/vectorstore.py (100%) rename {cyto_ml => src/cyto_ml}/models/__init__.py (100%) rename {cyto_ml => src/cyto_ml}/models/scivision.py (100%) rename {cyto_ml => tests}/tests/conftest.py (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1091.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_113.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_127.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_133.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1388.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1407.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1830.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1876.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_188.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1887.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1890.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1892.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1901.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1909.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1912.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1914.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1915.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1919.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1922.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1924.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1948.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1953.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1962.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1965.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_1981.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2012.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2071.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2102.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2108.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2110.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2115.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2117.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2119.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2172.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_2715.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_36.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_3612.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_3814.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_4715.tif (100%) rename {cyto_ml => tests}/tests/fixtures/test_images/testymctestface_4961.tif (100%) rename {cyto_ml => tests}/tests/test_image_embeddings.py (100%) rename {cyto_ml => tests}/tests/test_object_store.py (100%) rename {cyto_ml => tests}/tests/test_prepare_image.py (100%) rename {cyto_ml => tests}/tests/test_vector_store.py (100%) diff --git a/.gitignore b/.gitignore index df46e5a..3cd8b01 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ **/__pycache__/ vectors/ *.ipynb +*.egg-info/ diff --git a/pyproject.toml b/pyproject.toml index 3bf726b..d0c09b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + [project] name = "cyto_ml" -version = "0.1" +version = "0.1.0" +requires-python = "==3.9.*" description = "This package supports the processing and analysis of plankton sample data" readme = "README.md" -requires-python = "==3.9.*" - -[tool.setuptools] -py-modules = [] [tool.jupytext] formats = "ipynb,md" diff --git a/cyto_ml/__init__.py b/src/cyto_ml/__init__.py similarity index 100% rename from cyto_ml/__init__.py rename to src/cyto_ml/__init__.py diff --git a/cyto_ml/data/__init__.py b/src/cyto_ml/data/__init__.py similarity index 100% rename from cyto_ml/data/__init__.py rename to src/cyto_ml/data/__init__.py diff --git a/cyto_ml/data/intake.py b/src/cyto_ml/data/intake.py similarity index 100% rename from cyto_ml/data/intake.py rename to src/cyto_ml/data/intake.py diff --git a/cyto_ml/data/s3.py b/src/cyto_ml/data/s3.py similarity index 100% rename from cyto_ml/data/s3.py rename to src/cyto_ml/data/s3.py diff --git a/cyto_ml/data/vectorstore.py b/src/cyto_ml/data/vectorstore.py similarity index 100% rename from cyto_ml/data/vectorstore.py rename to src/cyto_ml/data/vectorstore.py diff --git a/cyto_ml/models/__init__.py b/src/cyto_ml/models/__init__.py similarity index 100% rename from cyto_ml/models/__init__.py rename to src/cyto_ml/models/__init__.py diff --git a/cyto_ml/models/scivision.py b/src/cyto_ml/models/scivision.py similarity index 100% rename from cyto_ml/models/scivision.py rename to src/cyto_ml/models/scivision.py diff --git a/cyto_ml/tests/conftest.py b/tests/tests/conftest.py similarity index 100% rename from cyto_ml/tests/conftest.py rename to tests/tests/conftest.py diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1091.tif b/tests/tests/fixtures/test_images/testymctestface_1091.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1091.tif rename to tests/tests/fixtures/test_images/testymctestface_1091.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_113.tif b/tests/tests/fixtures/test_images/testymctestface_113.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_113.tif rename to tests/tests/fixtures/test_images/testymctestface_113.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_127.tif b/tests/tests/fixtures/test_images/testymctestface_127.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_127.tif rename to tests/tests/fixtures/test_images/testymctestface_127.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_133.tif b/tests/tests/fixtures/test_images/testymctestface_133.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_133.tif rename to tests/tests/fixtures/test_images/testymctestface_133.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1388.tif b/tests/tests/fixtures/test_images/testymctestface_1388.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1388.tif rename to tests/tests/fixtures/test_images/testymctestface_1388.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1407.tif b/tests/tests/fixtures/test_images/testymctestface_1407.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1407.tif rename to tests/tests/fixtures/test_images/testymctestface_1407.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1830.tif b/tests/tests/fixtures/test_images/testymctestface_1830.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1830.tif rename to tests/tests/fixtures/test_images/testymctestface_1830.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1876.tif b/tests/tests/fixtures/test_images/testymctestface_1876.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1876.tif rename to tests/tests/fixtures/test_images/testymctestface_1876.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_188.tif b/tests/tests/fixtures/test_images/testymctestface_188.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_188.tif rename to tests/tests/fixtures/test_images/testymctestface_188.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1887.tif b/tests/tests/fixtures/test_images/testymctestface_1887.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1887.tif rename to tests/tests/fixtures/test_images/testymctestface_1887.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1890.tif b/tests/tests/fixtures/test_images/testymctestface_1890.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1890.tif rename to tests/tests/fixtures/test_images/testymctestface_1890.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1892.tif b/tests/tests/fixtures/test_images/testymctestface_1892.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1892.tif rename to tests/tests/fixtures/test_images/testymctestface_1892.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1901.tif b/tests/tests/fixtures/test_images/testymctestface_1901.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1901.tif rename to tests/tests/fixtures/test_images/testymctestface_1901.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1909.tif b/tests/tests/fixtures/test_images/testymctestface_1909.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1909.tif rename to tests/tests/fixtures/test_images/testymctestface_1909.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1912.tif b/tests/tests/fixtures/test_images/testymctestface_1912.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1912.tif rename to tests/tests/fixtures/test_images/testymctestface_1912.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1914.tif b/tests/tests/fixtures/test_images/testymctestface_1914.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1914.tif rename to tests/tests/fixtures/test_images/testymctestface_1914.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1915.tif b/tests/tests/fixtures/test_images/testymctestface_1915.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1915.tif rename to tests/tests/fixtures/test_images/testymctestface_1915.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1919.tif b/tests/tests/fixtures/test_images/testymctestface_1919.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1919.tif rename to tests/tests/fixtures/test_images/testymctestface_1919.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1922.tif b/tests/tests/fixtures/test_images/testymctestface_1922.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1922.tif rename to tests/tests/fixtures/test_images/testymctestface_1922.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1924.tif b/tests/tests/fixtures/test_images/testymctestface_1924.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1924.tif rename to tests/tests/fixtures/test_images/testymctestface_1924.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1948.tif b/tests/tests/fixtures/test_images/testymctestface_1948.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1948.tif rename to tests/tests/fixtures/test_images/testymctestface_1948.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1953.tif b/tests/tests/fixtures/test_images/testymctestface_1953.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1953.tif rename to tests/tests/fixtures/test_images/testymctestface_1953.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1962.tif b/tests/tests/fixtures/test_images/testymctestface_1962.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1962.tif rename to tests/tests/fixtures/test_images/testymctestface_1962.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1965.tif b/tests/tests/fixtures/test_images/testymctestface_1965.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1965.tif rename to tests/tests/fixtures/test_images/testymctestface_1965.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_1981.tif b/tests/tests/fixtures/test_images/testymctestface_1981.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_1981.tif rename to tests/tests/fixtures/test_images/testymctestface_1981.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2012.tif b/tests/tests/fixtures/test_images/testymctestface_2012.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2012.tif rename to tests/tests/fixtures/test_images/testymctestface_2012.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2071.tif b/tests/tests/fixtures/test_images/testymctestface_2071.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2071.tif rename to tests/tests/fixtures/test_images/testymctestface_2071.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2102.tif b/tests/tests/fixtures/test_images/testymctestface_2102.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2102.tif rename to tests/tests/fixtures/test_images/testymctestface_2102.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2108.tif b/tests/tests/fixtures/test_images/testymctestface_2108.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2108.tif rename to tests/tests/fixtures/test_images/testymctestface_2108.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2110.tif b/tests/tests/fixtures/test_images/testymctestface_2110.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2110.tif rename to tests/tests/fixtures/test_images/testymctestface_2110.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2115.tif b/tests/tests/fixtures/test_images/testymctestface_2115.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2115.tif rename to tests/tests/fixtures/test_images/testymctestface_2115.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2117.tif b/tests/tests/fixtures/test_images/testymctestface_2117.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2117.tif rename to tests/tests/fixtures/test_images/testymctestface_2117.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2119.tif b/tests/tests/fixtures/test_images/testymctestface_2119.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2119.tif rename to tests/tests/fixtures/test_images/testymctestface_2119.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2172.tif b/tests/tests/fixtures/test_images/testymctestface_2172.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2172.tif rename to tests/tests/fixtures/test_images/testymctestface_2172.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_2715.tif b/tests/tests/fixtures/test_images/testymctestface_2715.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_2715.tif rename to tests/tests/fixtures/test_images/testymctestface_2715.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_36.tif b/tests/tests/fixtures/test_images/testymctestface_36.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_36.tif rename to tests/tests/fixtures/test_images/testymctestface_36.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_3612.tif b/tests/tests/fixtures/test_images/testymctestface_3612.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_3612.tif rename to tests/tests/fixtures/test_images/testymctestface_3612.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_3814.tif b/tests/tests/fixtures/test_images/testymctestface_3814.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_3814.tif rename to tests/tests/fixtures/test_images/testymctestface_3814.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_4715.tif b/tests/tests/fixtures/test_images/testymctestface_4715.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_4715.tif rename to tests/tests/fixtures/test_images/testymctestface_4715.tif diff --git a/cyto_ml/tests/fixtures/test_images/testymctestface_4961.tif b/tests/tests/fixtures/test_images/testymctestface_4961.tif similarity index 100% rename from cyto_ml/tests/fixtures/test_images/testymctestface_4961.tif rename to tests/tests/fixtures/test_images/testymctestface_4961.tif diff --git a/cyto_ml/tests/test_image_embeddings.py b/tests/tests/test_image_embeddings.py similarity index 100% rename from cyto_ml/tests/test_image_embeddings.py rename to tests/tests/test_image_embeddings.py diff --git a/cyto_ml/tests/test_object_store.py b/tests/tests/test_object_store.py similarity index 100% rename from cyto_ml/tests/test_object_store.py rename to tests/tests/test_object_store.py diff --git a/cyto_ml/tests/test_prepare_image.py b/tests/tests/test_prepare_image.py similarity index 100% rename from cyto_ml/tests/test_prepare_image.py rename to tests/tests/test_prepare_image.py diff --git a/cyto_ml/tests/test_vector_store.py b/tests/tests/test_vector_store.py similarity index 100% rename from cyto_ml/tests/test_vector_store.py rename to tests/tests/test_vector_store.py From 9600b339cedf73bbcf854fdb275111d06cc9176e Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:48:26 +0100 Subject: [PATCH 03/13] undo accidental nesting of test folder --- tests/{tests => }/conftest.py | 0 .../fixtures/test_images/testymctestface_1091.tif | Bin .../fixtures/test_images/testymctestface_113.tif | Bin .../fixtures/test_images/testymctestface_127.tif | Bin .../fixtures/test_images/testymctestface_133.tif | Bin .../fixtures/test_images/testymctestface_1388.tif | Bin .../fixtures/test_images/testymctestface_1407.tif | Bin .../fixtures/test_images/testymctestface_1830.tif | Bin .../fixtures/test_images/testymctestface_1876.tif | Bin .../fixtures/test_images/testymctestface_188.tif | Bin .../fixtures/test_images/testymctestface_1887.tif | Bin .../fixtures/test_images/testymctestface_1890.tif | Bin .../fixtures/test_images/testymctestface_1892.tif | Bin .../fixtures/test_images/testymctestface_1901.tif | Bin .../fixtures/test_images/testymctestface_1909.tif | Bin .../fixtures/test_images/testymctestface_1912.tif | Bin .../fixtures/test_images/testymctestface_1914.tif | Bin .../fixtures/test_images/testymctestface_1915.tif | Bin .../fixtures/test_images/testymctestface_1919.tif | Bin .../fixtures/test_images/testymctestface_1922.tif | Bin .../fixtures/test_images/testymctestface_1924.tif | Bin .../fixtures/test_images/testymctestface_1948.tif | Bin .../fixtures/test_images/testymctestface_1953.tif | Bin .../fixtures/test_images/testymctestface_1962.tif | Bin .../fixtures/test_images/testymctestface_1965.tif | Bin .../fixtures/test_images/testymctestface_1981.tif | Bin .../fixtures/test_images/testymctestface_2012.tif | Bin .../fixtures/test_images/testymctestface_2071.tif | Bin .../fixtures/test_images/testymctestface_2102.tif | Bin .../fixtures/test_images/testymctestface_2108.tif | Bin .../fixtures/test_images/testymctestface_2110.tif | Bin .../fixtures/test_images/testymctestface_2115.tif | Bin .../fixtures/test_images/testymctestface_2117.tif | Bin .../fixtures/test_images/testymctestface_2119.tif | Bin .../fixtures/test_images/testymctestface_2172.tif | Bin .../fixtures/test_images/testymctestface_2715.tif | Bin .../fixtures/test_images/testymctestface_36.tif | Bin .../fixtures/test_images/testymctestface_3612.tif | Bin .../fixtures/test_images/testymctestface_3814.tif | Bin .../fixtures/test_images/testymctestface_4715.tif | Bin .../fixtures/test_images/testymctestface_4961.tif | Bin tests/{tests => }/test_image_embeddings.py | 0 tests/{tests => }/test_object_store.py | 0 tests/{tests => }/test_prepare_image.py | 0 tests/{tests => }/test_vector_store.py | 0 45 files changed, 0 insertions(+), 0 deletions(-) rename tests/{tests => }/conftest.py (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1091.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_113.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_127.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_133.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1388.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1407.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1830.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1876.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_188.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1887.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1890.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1892.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1901.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1909.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1912.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1914.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1915.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1919.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1922.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1924.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1948.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1953.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1962.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1965.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_1981.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2012.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2071.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2102.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2108.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2110.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2115.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2117.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2119.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2172.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_2715.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_36.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_3612.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_3814.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_4715.tif (100%) rename tests/{tests => }/fixtures/test_images/testymctestface_4961.tif (100%) rename tests/{tests => }/test_image_embeddings.py (100%) rename tests/{tests => }/test_object_store.py (100%) rename tests/{tests => }/test_prepare_image.py (100%) rename tests/{tests => }/test_vector_store.py (100%) diff --git a/tests/tests/conftest.py b/tests/conftest.py similarity index 100% rename from tests/tests/conftest.py rename to tests/conftest.py diff --git a/tests/tests/fixtures/test_images/testymctestface_1091.tif b/tests/fixtures/test_images/testymctestface_1091.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1091.tif rename to tests/fixtures/test_images/testymctestface_1091.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_113.tif b/tests/fixtures/test_images/testymctestface_113.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_113.tif rename to tests/fixtures/test_images/testymctestface_113.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_127.tif b/tests/fixtures/test_images/testymctestface_127.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_127.tif rename to tests/fixtures/test_images/testymctestface_127.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_133.tif b/tests/fixtures/test_images/testymctestface_133.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_133.tif rename to tests/fixtures/test_images/testymctestface_133.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1388.tif b/tests/fixtures/test_images/testymctestface_1388.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1388.tif rename to tests/fixtures/test_images/testymctestface_1388.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1407.tif b/tests/fixtures/test_images/testymctestface_1407.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1407.tif rename to tests/fixtures/test_images/testymctestface_1407.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1830.tif b/tests/fixtures/test_images/testymctestface_1830.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1830.tif rename to tests/fixtures/test_images/testymctestface_1830.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1876.tif b/tests/fixtures/test_images/testymctestface_1876.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1876.tif rename to tests/fixtures/test_images/testymctestface_1876.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_188.tif b/tests/fixtures/test_images/testymctestface_188.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_188.tif rename to tests/fixtures/test_images/testymctestface_188.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1887.tif b/tests/fixtures/test_images/testymctestface_1887.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1887.tif rename to tests/fixtures/test_images/testymctestface_1887.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1890.tif b/tests/fixtures/test_images/testymctestface_1890.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1890.tif rename to tests/fixtures/test_images/testymctestface_1890.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1892.tif b/tests/fixtures/test_images/testymctestface_1892.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1892.tif rename to tests/fixtures/test_images/testymctestface_1892.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1901.tif b/tests/fixtures/test_images/testymctestface_1901.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1901.tif rename to tests/fixtures/test_images/testymctestface_1901.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1909.tif b/tests/fixtures/test_images/testymctestface_1909.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1909.tif rename to tests/fixtures/test_images/testymctestface_1909.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1912.tif b/tests/fixtures/test_images/testymctestface_1912.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1912.tif rename to tests/fixtures/test_images/testymctestface_1912.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1914.tif b/tests/fixtures/test_images/testymctestface_1914.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1914.tif rename to tests/fixtures/test_images/testymctestface_1914.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1915.tif b/tests/fixtures/test_images/testymctestface_1915.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1915.tif rename to tests/fixtures/test_images/testymctestface_1915.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1919.tif b/tests/fixtures/test_images/testymctestface_1919.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1919.tif rename to tests/fixtures/test_images/testymctestface_1919.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1922.tif b/tests/fixtures/test_images/testymctestface_1922.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1922.tif rename to tests/fixtures/test_images/testymctestface_1922.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1924.tif b/tests/fixtures/test_images/testymctestface_1924.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1924.tif rename to tests/fixtures/test_images/testymctestface_1924.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1948.tif b/tests/fixtures/test_images/testymctestface_1948.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1948.tif rename to tests/fixtures/test_images/testymctestface_1948.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1953.tif b/tests/fixtures/test_images/testymctestface_1953.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1953.tif rename to tests/fixtures/test_images/testymctestface_1953.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1962.tif b/tests/fixtures/test_images/testymctestface_1962.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1962.tif rename to tests/fixtures/test_images/testymctestface_1962.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1965.tif b/tests/fixtures/test_images/testymctestface_1965.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1965.tif rename to tests/fixtures/test_images/testymctestface_1965.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_1981.tif b/tests/fixtures/test_images/testymctestface_1981.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_1981.tif rename to tests/fixtures/test_images/testymctestface_1981.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2012.tif b/tests/fixtures/test_images/testymctestface_2012.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2012.tif rename to tests/fixtures/test_images/testymctestface_2012.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2071.tif b/tests/fixtures/test_images/testymctestface_2071.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2071.tif rename to tests/fixtures/test_images/testymctestface_2071.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2102.tif b/tests/fixtures/test_images/testymctestface_2102.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2102.tif rename to tests/fixtures/test_images/testymctestface_2102.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2108.tif b/tests/fixtures/test_images/testymctestface_2108.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2108.tif rename to tests/fixtures/test_images/testymctestface_2108.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2110.tif b/tests/fixtures/test_images/testymctestface_2110.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2110.tif rename to tests/fixtures/test_images/testymctestface_2110.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2115.tif b/tests/fixtures/test_images/testymctestface_2115.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2115.tif rename to tests/fixtures/test_images/testymctestface_2115.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2117.tif b/tests/fixtures/test_images/testymctestface_2117.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2117.tif rename to tests/fixtures/test_images/testymctestface_2117.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2119.tif b/tests/fixtures/test_images/testymctestface_2119.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2119.tif rename to tests/fixtures/test_images/testymctestface_2119.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2172.tif b/tests/fixtures/test_images/testymctestface_2172.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2172.tif rename to tests/fixtures/test_images/testymctestface_2172.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_2715.tif b/tests/fixtures/test_images/testymctestface_2715.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_2715.tif rename to tests/fixtures/test_images/testymctestface_2715.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_36.tif b/tests/fixtures/test_images/testymctestface_36.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_36.tif rename to tests/fixtures/test_images/testymctestface_36.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_3612.tif b/tests/fixtures/test_images/testymctestface_3612.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_3612.tif rename to tests/fixtures/test_images/testymctestface_3612.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_3814.tif b/tests/fixtures/test_images/testymctestface_3814.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_3814.tif rename to tests/fixtures/test_images/testymctestface_3814.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_4715.tif b/tests/fixtures/test_images/testymctestface_4715.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_4715.tif rename to tests/fixtures/test_images/testymctestface_4715.tif diff --git a/tests/tests/fixtures/test_images/testymctestface_4961.tif b/tests/fixtures/test_images/testymctestface_4961.tif similarity index 100% rename from tests/tests/fixtures/test_images/testymctestface_4961.tif rename to tests/fixtures/test_images/testymctestface_4961.tif diff --git a/tests/tests/test_image_embeddings.py b/tests/test_image_embeddings.py similarity index 100% rename from tests/tests/test_image_embeddings.py rename to tests/test_image_embeddings.py diff --git a/tests/tests/test_object_store.py b/tests/test_object_store.py similarity index 100% rename from tests/tests/test_object_store.py rename to tests/test_object_store.py diff --git a/tests/tests/test_prepare_image.py b/tests/test_prepare_image.py similarity index 100% rename from tests/tests/test_prepare_image.py rename to tests/test_prepare_image.py diff --git a/tests/tests/test_vector_store.py b/tests/test_vector_store.py similarity index 100% rename from tests/tests/test_vector_store.py rename to tests/test_vector_store.py From 5807fede04168e97db5d813133bd7946f411932d Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:57:53 +0100 Subject: [PATCH 04/13] run black --- pyproject.toml | 9 +++++++++ scripts/intake_metadata.py | 1 + 2 files changed, 10 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d0c09b0..ffd6870 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,12 @@ readme = "README.md" [tool.jupytext] formats = "ipynb,md" + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore::DeprecationWarning", +] + +[tool.black] +target-version = ["py39"] +line-length = 88 diff --git a/scripts/intake_metadata.py b/scripts/intake_metadata.py index 69e4516..8c2bba5 100644 --- a/scripts/intake_metadata.py +++ b/scripts/intake_metadata.py @@ -7,6 +7,7 @@ Via https://gallery.pangeo.io/repos/pangeo-data/pangeo-tutorial-gallery/intake.html#Build-an-intake-catalog """ + import os from cyto_ml.data.intake import intake_yaml from cyto_ml.data.s3 import s3_endpoint, image_index From 49b1d21c258f9da3398e13fd60f3751dcf1596c1 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:58:43 +0100 Subject: [PATCH 05/13] change python -m pytest to pytest in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 731797f..6c87f6d 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ Please note that this is specifically pinned to python 3.9 due to dependency ver Get started by cloning this repository and running -`pip install -e .` +`python -m pip install -e .` ### Running tests -`python -m pytest` or `py.test` +`pytest` or `py.test` ## Contents From 70b82c228c47ea9c845d122e05e2c2d19e4afcac Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Fri, 23 Aug 2024 22:21:40 +0100 Subject: [PATCH 06/13] update python version; deps in pyproject --- .python-version | 1 + pyproject.toml | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .python-version diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/pyproject.toml b/pyproject.toml index ffd6870..e9449c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,20 @@ build-backend = "setuptools.build_meta" [project] name = "cyto_ml" -version = "0.1.0" -requires-python = "==3.9.*" +version = "0.2.0" +requires-python = ">=3.12" description = "This package supports the processing and analysis of plankton sample data" readme = "README.md" +dependencies = [ + "chromadb", + "python-dotenv", + "s3fs", + "torch", +] + +[project.optional-dependencies] +jupyter = ["ipython", "jupyterlab", "jupytext"] +dev = ["pytest", "black", "flake8", "isort"] [tool.jupytext] formats = "ipynb,md" @@ -18,5 +28,5 @@ filterwarnings = [ ] [tool.black] -target-version = ["py39"] +target-version = ["py312"] line-length = 88 From bc7c43bc989aaa07c42f9d870b519b3f6888a77d Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:08:56 +0100 Subject: [PATCH 07/13] load scivision model from forked repo --- .gitignore | 1 + pyproject.toml | 6 +++++ src/cyto_ml/models/scivision.py | 39 ++++++++------------------------- tests/conftest.py | 9 +++----- tests/test_image_embeddings.py | 1 - 5 files changed, 19 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 3cd8b01..c5a8c5f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ vectors/ *.ipynb *.egg-info/ +venv/ diff --git a/pyproject.toml b/pyproject.toml index e9449c3..5f66ca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,9 +10,15 @@ description = "This package supports the processing and analysis of plankton sam readme = "README.md" dependencies = [ "chromadb", + "intake==0.7.0", + "intake-xarray", + "pandas", "python-dotenv", "s3fs", + "scikit-image", # secretly required by intake-xarray as default reader "torch", + "xarray", + "resnet50-cefas@git+https://github.com/jmarshrossney/resnet50-cefas", ] [project.optional-dependencies] diff --git a/src/cyto_ml/models/scivision.py b/src/cyto_ml/models/scivision.py index efba243..6a6e829 100644 --- a/src/cyto_ml/models/scivision.py +++ b/src/cyto_ml/models/scivision.py @@ -1,34 +1,7 @@ -from scivision import load_pretrained_model -from scivision.io import PretrainedModel import torch -import torchvision +from torchvision.transforms.v2.functional import to_image, to_dtype from xarray import DataArray -SCIVISION_URL = ( - "https://github.com/alan-turing-institute/plankton-cefas-scivision" # noqa: E501 -) - - -def load_model(url: str): - """Load a scivision model from a URL, for example - https://github.com/alan-turing-institute/plankton-cefas-scivision - """ - model = load_pretrained_model(url) - return model - - -def truncate_model(model: PretrainedModel): - """ - Accepts a scivision model wrapper and returns the pytorch model, - with its last fully-connected layer removed so that it returns - 2048 features rather than a handle of label predictions - """ - network = torch.nn.Sequential( - *(list(model._plumbing.model.pretrained_model.children())[:-1]) - ) - return network - - def prepare_image(image: DataArray): """ Take an xarray of image data and prepare it to pass through the model @@ -39,7 +12,12 @@ def prepare_image(image: DataArray): image_numpy = image.to_numpy() # Convert the image data to a PyTorch tensor - tensor_image = torchvision.transforms.ToTensor()(image_numpy) + tensor_image = to_dtype( + to_image(image_numpy), # permutes HWC -> CHW + torch.float32, + scale=True, # rescales [0, 255] -> [0, 1] + ) + assert torch.all((tensor_image >= 0.0) & (tensor_image <= 1.0)) # Check if the input is a single image or a batch if len(tensor_image.shape) == 3: @@ -52,4 +30,5 @@ def prepare_image(image: DataArray): def flat_embeddings(features: torch.Tensor): """Utility function that takes the features returned by the model in truncate_model And flattens them into a list suitable for storing in a vector database""" - return list(features[0].squeeze(1).squeeze(1).detach().numpy().astype(float)) + # TODO: this only returns the 0th tensor in the batch...why? + return features[0].detach().tolist() diff --git a/tests/conftest.py b/tests/conftest.py index 1b092ae..9cda323 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,7 @@ import os import pytest -from cyto_ml.models.scivision import ( - load_model, - truncate_model, - SCIVISION_URL, -) + +from resnet50_cefas import load_model @pytest.fixture @@ -30,7 +27,7 @@ def image_batch(image_dir): @pytest.fixture def scivision_model(): - return truncate_model(load_model(SCIVISION_URL)) + return load_model(strip_final_layer=True) @pytest.fixture diff --git a/tests/test_image_embeddings.py b/tests/test_image_embeddings.py index 58263b8..e7c961a 100644 --- a/tests/test_image_embeddings.py +++ b/tests/test_image_embeddings.py @@ -2,7 +2,6 @@ from torch import Tensor from cyto_ml.models.scivision import prepare_image, flat_embeddings - def test_embeddings(scivision_model, single_image): features = scivision_model(prepare_image(ImageSource(single_image).to_dask())) From 6c7aaf2dd8865a26abd834755fd971745e282f17 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:38:42 +0100 Subject: [PATCH 08/13] move vector store; minor adjustments --- src/cyto_ml/data/vectorstore.py | 2 +- src/cyto_ml/models/scivision.py | 5 +++-- tests/test_prepare_image.py | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cyto_ml/data/vectorstore.py b/src/cyto_ml/data/vectorstore.py index f6e5bcc..08cc531 100644 --- a/src/cyto_ml/data/vectorstore.py +++ b/src/cyto_ml/data/vectorstore.py @@ -9,7 +9,7 @@ logging.basicConfig(level=logging.INFO) # TODO make this sensibly configurable, not confusingly hardcoded -STORE = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../vectors") +STORE = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../../vectors") client = chromadb.PersistentClient( path=STORE, diff --git a/src/cyto_ml/models/scivision.py b/src/cyto_ml/models/scivision.py index 6a6e829..a9f8f05 100644 --- a/src/cyto_ml/models/scivision.py +++ b/src/cyto_ml/models/scivision.py @@ -19,11 +19,12 @@ def prepare_image(image: DataArray): ) assert torch.all((tensor_image >= 0.0) & (tensor_image <= 1.0)) - # Check if the input is a single image or a batch - if len(tensor_image.shape) == 3: + if tensor_image.dim() == 3: # Single image, add a batch dimension tensor_image = tensor_image.unsqueeze(0) + assert tensor_image.dim() == 4 + return tensor_image diff --git a/tests/test_prepare_image.py b/tests/test_prepare_image.py index 459496b..2c602da 100644 --- a/tests/test_prepare_image.py +++ b/tests/test_prepare_image.py @@ -9,8 +9,8 @@ def test_single_image(single_image): - image_data = ImageSource(single_image).to_dask() + # Tensorise the image (potentially normalise if we have useful values) prepared_image = prepare_image(image_data) @@ -25,7 +25,6 @@ def test_image_batch(image_batch): We either pad them (and process a lot of blank space) or stick to single image input """ # Load a batch of plankton images - image_data = ImageSource(image_batch).to_dask() with pytest.raises(ValueError) as err: From c03d9f2b2cb9e6ae83e16fcab8e3bdc788bb545a Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:47:12 +0100 Subject: [PATCH 09/13] test install/setup using pip and conda --- README.md | 38 +++++++++++++++++++++++++++++--------- environment.yml | 25 ++++++++++++++----------- pyproject.toml | 3 ++- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6c87f6d..fa09156 100644 --- a/README.md +++ b/README.md @@ -6,26 +6,46 @@ It's a companion project to an R-shiny based image annotation app that is not ye ## Installation -### Python environment setup +### Environment and package installation -Use anaconda or miniconda to create a python environment using the included `environment.yml` +#### Using pip + +Create a fresh virtual environment in the repository root using Python >=3.12 and (e.g.) `venv`: ``` -conda env create -f environment.yml +python -m venv venv ``` -Please note that this is specifically pinned to python 3.9 due to dependency versions; we make experimental use of the [CEFAS plankton model available through SciVision](https://sci.vision/#/model/resnet50-plankton), which in turn uses an older version of pytorch that isn't packaged above python 3.9. +Next, install the package using `pip`: -### Object store connection +``` +python -m pip install . +``` -`.env` contains environment variable names for S3 connection details for the [JASMIN object store](https://github.com/NERC-CEH/object_store_tutorial/). Fill these in with your own credentials. If you're not sure what the `ENDPOINT` should be, please reach out to one of the project contributors listed below. +Most likely you are interested in developing and/or experimenting, so you will probably want to install the package in 'editable' mode (`-e`), along with dev tools and jupyter notebook functionality +``` +python -m pip install -e .[all] +``` -### Package installation +#### Using conda -Get started by cloning this repository and running +Use anaconda or miniconda to create a python environment using the included `environment.yml` -`python -m pip install -e .` +``` +conda env create -f environment.yml +conda activate cyto_ml +``` + +Next install this package _without dependencies_: + +``` +python -m pip install --no-deps -e . +``` + +### Object store connection + +`.env` contains environment variable names for S3 connection details for the [JASMIN object store](https://github.com/NERC-CEH/object_store_tutorial/). Fill these in with your own credentials. If you're not sure what the `ENDPOINT` should be, please reach out to one of the project contributors listed below. ### Running tests diff --git a/environment.yml b/environment.yml index 9bccdc0..8c596a6 100644 --- a/environment.yml +++ b/environment.yml @@ -1,22 +1,25 @@ -name: cyto_39 +name: cyto_ml channels: - pytorch - conda-forge - - defaults +channel_priority: flexible dependencies: - - python=3.9 - - pytorch=1.10.0 - - mkl=2024.0 - - chromadb=0.5.3 + - python=3.12 + - pytorch + - black + - chromadb + - flake8 - intake-xarray - - scikit-image + - intake=0.7 + - isort + - jupyterlab + - jupytext - pandas - pytest - python-dotenv - s3fs - - jupyterlab - - jupytext + - scikit-image + - xarray - pip - pip: - - scivision - - git+https://github.com/alan-turing-institute/plankton-cefas-scivision@main + - git+https://github.com/jmarshrossney/resnet50-cefas diff --git a/pyproject.toml b/pyproject.toml index 5f66ca0..e8bc480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,9 @@ dependencies = [ ] [project.optional-dependencies] -jupyter = ["ipython", "jupyterlab", "jupytext"] +jupyter = ["jupyterlab", "jupytext"] dev = ["pytest", "black", "flake8", "isort"] +all = ["cyto_ml[jupyter,dev]"] [tool.jupytext] formats = "ipynb,md" From d84c768a47e7e1754a15357a6cc6f25826e7af81 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 15:33:57 +0100 Subject: [PATCH 10/13] minimally remove scivision from ImageEmbeddings notebook --- notebooks/ImageEmbeddings.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/notebooks/ImageEmbeddings.md b/notebooks/ImageEmbeddings.md index bb9f13a..19e2e8d 100644 --- a/notebooks/ImageEmbeddings.md +++ b/notebooks/ImageEmbeddings.md @@ -5,7 +5,7 @@ jupyter: extension: .md format_name: markdown format_version: '1.3' - jupytext_version: 1.16.3 + jupytext_version: 1.16.4 kernelspec: display_name: Python 3 (ipykernel) language: python @@ -19,7 +19,6 @@ Use this with the `cyto_39` environment (the scivision model needs a specific ve ```python import os -from scivision import load_pretrained_model, load_dataset from dotenv import load_dotenv import torch import torchvision @@ -29,11 +28,15 @@ sys.path.append('../') from cyto_ml.models.scivision import prepare_image from intake_xarray import ImageSource load_dotenv() # sets our object store endpoint and credentials from the .env file + +from intake import open_catalog + +from resnet50_cefas import load_model ``` ```python -dataset = load_dataset(f"{os.environ.get('ENDPOINT', '')}/metadata/intake.yml") -model = load_pretrained_model("https://github.com/alan-turing-institute/plankton-cefas-scivision") +dataset = open_catalog(f"{os.environ.get('ENDPOINT', '')}/metadata/intake.yml") +model = load_model() dataset.test_image().to_dask() ``` From daa6733ca438df02710f1e6045306c4c589f5451 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 16:15:33 +0100 Subject: [PATCH 11/13] update remainder of ImageEmbeddings.md --- notebooks/ImageEmbeddings.md | 41 ++++++++---------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/notebooks/ImageEmbeddings.md b/notebooks/ImageEmbeddings.md index 19e2e8d..19362ab 100644 --- a/notebooks/ImageEmbeddings.md +++ b/notebooks/ImageEmbeddings.md @@ -12,10 +12,10 @@ jupyter: name: python3 --- -Use this with the `cyto_39` environment (the scivision model needs a specific version of `pytorch` that isn't packaged for >3.9, i have raised a Github issue asking if they plan to update it) +Use this with the `cyto_ml` environment. `conda env create -f environment.yml` -`conda activate cyto_39` +`conda activate cyto_ml` ```python import os @@ -40,50 +40,25 @@ model = load_model() dataset.test_image().to_dask() ``` -The scivision wrapper depends on this being an xarray Dataset with settable attributes, rather than a DataArray - -Setting exif_tags: True (Dataset) or False (DataArray) is what controls this -https://docs.xarray.dev/en/stable/generated/xarray.DataArray.to_dataset.html - -https://github.com/alan-turing-institute/scivision/blob/07fb74e5231bc1d56cf39df38c19ef40e3265e4c/src/scivision/io/reader.py#L183 -https://github.com/intake/intake/blob/29c8878aa7bf6e93185e2c9639f8739445dff22b/intake/__init__.py#L101 - -But now we're dependent on image height and width metadata being set in the EXIF tags to use the `predict` interface, this is set in the model description through `scivision`, this is brittle - -https://github.com/alan-turing-institute/plankton-cefas-scivision/blob/main/resnet50_cefas/model.py#L71 - - - -A quick look at the example dataset that comes with the model, for reference - - -In this case we don't want to use the `predict` interface anyway (one of N class labels) - we want the features that go into the last fully-connected layer (as described here https://stackoverflow.com/a/52548419) - ```python -network = torch.nn.Sequential(*(list(model._plumbing.model.pretrained_model.children())[:-1])) +network = load_model(strip_final_layer=True) ``` ```python imgs = dataset.test_image().to_dask() -i= imgs.to_numpy() -i.shape - +imgs.to_numpy().shape ``` -https://github.com/alan-turing-institute/plankton-cefas-scivision/blob/main/resnet50_cefas/data.py - - - Pass the image through our truncated network and get some embeddings out ```python -o = torch.stack([torchvision.transforms.ToTensor()(i)]) +o = prepare_image(imgs) feats = network(o) feats.shape ``` ```python -embeddings = list(feats[0].squeeze(1).squeeze(1).detach().numpy().astype(float)) +embeddings = feats[0].tolist() ``` ```python @@ -132,7 +107,7 @@ index ```python def flat_embeddings(features: torch.Tensor): - return list(features[0].squeeze(1).squeeze(1).detach().numpy().astype(float)) + return features[0].tolist() ``` ```python @@ -161,6 +136,8 @@ This scales ok at 8000 or so images collection.count() ``` +This is _really_ slow - joe + ```python res = index.apply(file_embeddings, axis=1) ``` From 32bdc1faa49f0241ea4225899396c54e52ac11a8 Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sat, 24 Aug 2024 16:19:49 +0100 Subject: [PATCH 12/13] add matplotlib dependence for notebooks --- environment.yml | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8c596a6..1a019be 100644 --- a/environment.yml +++ b/environment.yml @@ -14,6 +14,7 @@ dependencies: - isort - jupyterlab - jupytext + - matplotlib - pandas - pytest - python-dotenv diff --git a/pyproject.toml b/pyproject.toml index e8bc480..bb89f2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ ] [project.optional-dependencies] -jupyter = ["jupyterlab", "jupytext"] +jupyter = ["jupyterlab", "jupytext", "matplotlib"] dev = ["pytest", "black", "flake8", "isort"] all = ["cyto_ml[jupyter,dev]"] From 2bf3586f3047dbf37831a6cadcfcb1b44c30b77d Mon Sep 17 00:00:00 2001 From: marshrossney <17361029+marshrossney@users.noreply.github.com> Date: Sun, 25 Aug 2024 10:16:50 +0100 Subject: [PATCH 13/13] run black; update flake8 config to ignore directories --- .flake8 | 5 +++++ src/cyto_ml/models/scivision.py | 1 + tests/test_image_embeddings.py | 1 + 3 files changed, 7 insertions(+) diff --git a/.flake8 b/.flake8 index aa079ec..ffe76ee 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,7 @@ [flake8] max-line-length=120 +exclude = + venv + __pycache__ + tests + vectors diff --git a/src/cyto_ml/models/scivision.py b/src/cyto_ml/models/scivision.py index a9f8f05..ec022e1 100644 --- a/src/cyto_ml/models/scivision.py +++ b/src/cyto_ml/models/scivision.py @@ -2,6 +2,7 @@ from torchvision.transforms.v2.functional import to_image, to_dtype from xarray import DataArray + def prepare_image(image: DataArray): """ Take an xarray of image data and prepare it to pass through the model diff --git a/tests/test_image_embeddings.py b/tests/test_image_embeddings.py index e7c961a..58263b8 100644 --- a/tests/test_image_embeddings.py +++ b/tests/test_image_embeddings.py @@ -2,6 +2,7 @@ from torch import Tensor from cyto_ml.models.scivision import prepare_image, flat_embeddings + def test_embeddings(scivision_model, single_image): features = scivision_model(prepare_image(ImageSource(single_image).to_dask()))