Sensorstoshow table column

documentation/changelog.rst

@@ -28,11 +28,13 @@ v0.23.0 | September 18, 2024
 
 .. note:: Read more on these features on `the FlexMeasures blog <https://flexmeasures.io/023-data-insights-and-white-labelling/>`_.
 
+.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
+
 New features
 -------------
 * New chart type on sensor page: histogram [see `PR #1143 <https://github.com/FlexMeasures/flexmeasures/pull/1143>`_]
 * Add basic sensor info to sensor page [see `PR #1115 <https://github.com/FlexMeasures/flexmeasures/pull/1115>`_]
-* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 <https://github.com/FlexMeasures/flexmeasures/pull/1200>`_]
+* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data from parent source(attributes field) [see `PR #1200 <https://github.com/FlexMeasures/flexmeasures/pull/1200>`_]
 * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors/<id>/stats` endpoint to get sensor statistics [see `PR #1116 <https://github.com/FlexMeasures/flexmeasures/pull/1116>`_]
 * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 <https://github.com/FlexMeasures/flexmeasures/pull/1125>`_ and `PR #1177 <https://github.com/FlexMeasures/flexmeasures/pull/1177>`_]
 * Support zoom-in action on the asset and sensor charts [see `PR #1130 <https://github.com/FlexMeasures/flexmeasures/pull/1130>`_]

flexmeasures/data/models/generic_assets.py

@@ -4,6 +4,7 @@
 from typing import Any
 import json
 
+from flask import current_app
 from flask_security import current_user
 import pandas as pd
 from sqlalchemy import select
@@ -23,7 +24,7 @@
 from flexmeasures.data.services.timerange import get_timerange
 from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER
 from flexmeasures.utils import geo_utils
-from flexmeasures.utils.coding_utils import flatten_unique, process_sensors
+from flexmeasures.utils.coding_utils import flatten_unique
 from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution
 
 
@@ -143,7 +144,6 @@ class GenericAsset(db.Model, AuthModelMixin):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        process_sensors(self)  # Migrate data to sensors_to_show
 
     def __acl__(self):
         """
@@ -170,6 +170,127 @@ def __repr__(self):
             self.generic_asset_type.name,
         )
 
+    def validate_sensors_to_show(
+        self,
+    ) -> list[dict[str, str | None | "Sensor" | list["Sensor"]]]:  # noqa: F821
+        """
+        Validate and transform the 'sensors_to_show' attribute into the latest format for use in graph-making code.
+
+        This function ensures that the 'sensors_to_show' attribute:
+        1. Follows the latest format, even if the data in the database uses an older format.
+        2. Contains only sensors that the user has access to (based on the current asset, account, or public availability).
+        3. Returns a list of dictionaries where each dictionary contains either a single sensor or a group of sensors with an optional title.
+
+        Steps:
+        - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly.
+        - It checks if each sensor is accessible by the user and filters out any unauthorized sensors.
+        - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles.
+
+        Details on format:
+        - The 'sensors_to_show' attribute is defined as a list of sensor IDs or nested lists of sensor IDs (to indicate grouping).
+        - Titles may be associated with rows of sensors. If no title is provided, `{"title": None}` will be assigned.
+        - Nested lists of sensors indicate that they should be shown together (e.g., layered in the same chart).
+
+        Example inputs:
+        1. Simple list of sensors, where sensors 42 and 44 are grouped:
+            sensors_to_show = [40, 35, 41, [42, 44], 43, 45]
+
+        2. List with titles and sensor groupings:
+            sensors_to_show = [
+                {"title": "Title 1", "sensor": 40},
+                {"title": "Title 2", "sensors": [41, 42]},
+                [43, 44], 45, 46
+            ]
+
+        Returned structure:
+        - The function returns a list of dictionaries, with each dictionary containing either a 'sensor' (for individual sensors) or 'sensors' (for groups of sensors), and an optional 'title'.
+        - Example output:
+            [
+                {"title": "Title 1", "sensor": <Sensor object for sensor 40>},
+                {"title": "Title 2", "sensors": [<Sensor object for sensor 41>, <Sensor object for sensor 42>]},
+                {"title": None, "sensors": [<Sensor object for sensor 43>, <Sensor object for sensor 44>]},
+                {"title": None, "sensor": <Sensor object for sensor 45>},
+                {"title": None, "sensor": <Sensor object for sensor 46>}
+            ]
+
+        If the 'sensors_to_show' attribute is missing, the function defaults to showing two of the asset's sensors, grouped together if they share the same unit, or separately if not.
+
+        Unauthorized sensors are filtered out, and a warning is logged. Only sensors the user has permission to access are included in the final result.
+        """
+
+        if self.attributes is not None:
+            self.sensors_to_show = self.attributes.get("sensors_to_show", [])
+        else:
+            self.sensors_to_show = []
+
+        if not self.has_attribute("sensors_to_show"):
+            sensors_to_show = self.sensors[:2]
+            if (
+                len(sensors_to_show) == 2
+                and sensors_to_show[0].unit == sensors_to_show[1].unit
+            ):
+                # Sensors are shown together (e.g. they can share the same y-axis)
+                return [{"title": None, "sensors": sensors_to_show}]
+            # Otherwise, show separately
+            return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show]
+
+        sensor_ids_to_show = self.get_attribute("sensors_to_show")
+        # Import the schema for validation
+        from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema
+
+        sensors_to_show_schema = SensorsToShowSchema()
+
+        # Deserialize the sensor_ids_to_show using SensorsToShowSchema
+        standardized_sensors_to_show = sensors_to_show_schema.deserialize(
+            sensor_ids_to_show
+        )
+
+        sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show)
+
+        # Only allow showing sensors from assets owned by the user's organization,
+        # except in play mode, where any sensor may be shown
+        accounts = [self.owner] if self.owner is not None else None
+        if current_app.config.get("FLEXMEASURES_MODE") == "play":
+            from flexmeasures.data.models.user import Account
+
+            accounts = db.session.scalars(select(Account)).all()
+
+        from flexmeasures.data.services.sensors import get_sensors
+
+        accessible_sensor_map = {
+            sensor.id: sensor
+            for sensor in get_sensors(
+                account=accounts,
+                include_public_assets=True,
+                sensor_id_allowlist=sensor_id_allowlist,
+            )
+        }
+
+        # Build list of sensor objects that are accessible
+        sensors_to_show = []
+        missed_sensor_ids = []
+
+        for entry in standardized_sensors_to_show:
+
+            title = entry.get("title")
+            sensors = entry.get("sensors")
+
+            accessible_sensors = [
+                accessible_sensor_map.get(sid)
+                for sid in sensors
+                if sid in accessible_sensor_map
+            ]
+            inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map]
+            missed_sensor_ids.extend(inaccessible)
+            if accessible_sensors:
+                sensors_to_show.append({"title": title, "sensors": accessible_sensors})
+
+        if missed_sensor_ids:
+            current_app.logger.warning(
+                f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}."
+            )
+        return sensors_to_show
+
     @property
     def asset_type(self) -> GenericAssetType:
         """This property prepares for dropping the "generic" prefix later"""
@@ -458,7 +579,7 @@ def chart(
         :param resolution: optionally set the resolution of data being displayed
         :returns: JSON string defining vega-lite chart specs
         """
-        processed_sensors_to_show = process_sensors(self)
+        processed_sensors_to_show = self.validate_sensors_to_show()
         sensors = flatten_unique(processed_sensors_to_show)
         for sensor in sensors:
             sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name)
@@ -641,7 +762,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]:
                       'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
                   }
         """
-        return self.get_timerange(process_sensors(self))
+        return self.get_timerange(self.validate_sensors_to_show())
 
     @classmethod
     def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]:  # noqa F821

flexmeasures/ui/crud/assets/views.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import json
 from flask import url_for, current_app, request
 from flask_classful import FlaskView, route
 from flask_security import login_required, current_user
@@ -113,6 +114,14 @@ def get(self, id: str, **kwargs):
 
         get_asset_response = InternalApi().get(url_for("AssetAPI:fetch_one", id=id))
         asset_dict = get_asset_response.json()
+
+        # set sensors to show to list from string, this is not currently being used on the frontend from my knowledge
+        # it may be better popped of instead, but will leave it hear for now
+        if asset_dict.get("sensors_to_show") and not isinstance(
+            asset_dict.get("sensors_to_show"), list
+        ):
+            asset_dict["sensors_to_show"] = json.loads(asset_dict["sensors_to_show"])
+
         asset = process_internal_api_response(asset_dict, int(id), make_obj=True)
 
         asset_form = AssetForm()

flexmeasures/utils/coding_utils.py

@@ -8,11 +8,7 @@
 import importlib
 import pkgutil
 
-from sqlalchemy import select
 from flask import current_app
-from flask_security import current_user
-
-from flexmeasures.data import db
 
 
 def delete_key_recursive(value, key):
@@ -181,128 +177,3 @@ def find_classes_modules(module, superclass, skiptest=True):
 
 def get_classes_module(module, superclass, skiptest=True) -> dict:
     return dict(find_classes_modules(module, superclass, skiptest=skiptest))
-
-
-def process_sensors(asset) -> list[dict[str, "Sensor"]]:  # noqa F821
-    """
-    Sensors to show, as defined by the sensors_to_show attribute.
-
-    Sensors to show are defined as a list of sensor IDs, which are set by the "sensors_to_show" field in the asset's "attributes" column.
-    Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets.
-    In play mode, sensors from different accounts can be added.
-
-    Sensor IDs can be nested to denote that sensors should be 'shown together', for example, layered rather than vertically concatenated.
-    Additionally, each row of sensors can be accompanied by a title.
-    If no title is provided, `"title": None` will be assigned in the returned dictionary.
-
-    How to interpret 'shown together' is technically left up to the function returning chart specifications, as are any restrictions regarding which sensors can be shown together, such as:
-    - Whether they should share the same unit
-    - Whether they should share the same name
-    - Whether they should belong to different assets
-
-    For example, this input denotes showing sensors 42 and 44 together:
-
-        sensors_to_show = [40, 35, 41, [42, 44], 43, 45]
-
-    And this input denotes showing sensors 42 and 44 together with a custom title:
-
-        sensors_to_show = [
-            {"title": "Title 1", "sensor": 40},
-            {"title": "Title 2", "sensors": [41, 42]},
-            [43, 44], 45, 46
-        ]
-
-    In both cases, the returned format will contain sensor objects mapped to their respective sensor IDs, as follows:
-
-        [
-            {"title": "Title 1", "sensor": <Sensor object for sensor 40>},
-            {"title": "Title 2", "sensors": [<Sensor object for sensor 41>, <Sensor object for sensor 42>]},
-            {"title": None, "sensors": [<Sensor object for sensor 43>, <Sensor object for sensor 44>]},
-            {"title": None, "sensor": <Sensor object for sensor 45>},
-            {"title": None, "sensor": <Sensor object for sensor 46>}
-        ]
-
-    In case the `sensors_to_show` field is missing, it defaults to two of the asset's sensors. These will be shown together (e.g., sharing the same y-axis) if they share the same unit; otherwise, they will be shown separately.
-
-    Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view.
-    """
-    old_sensors_to_show = (
-        asset.sensors_to_show
-    )  # Used to check if sensors_to_show was updated
-    if asset.attributes is not None:
-        asset.sensors_to_show = asset.attributes.get("sensors_to_show", [])
-    else:
-        asset.sensors_to_show = []
-
-    if not asset.sensors_to_show or asset.sensors_to_show == {}:
-        sensors_to_show = asset.sensors[:2]
-        if (
-            len(sensors_to_show) == 2
-            and sensors_to_show[0].unit == sensors_to_show[1].unit
-        ):
-            # Sensors are shown together (e.g. they can share the same y-axis)
-            return [{"title": None, "sensors": sensors_to_show}]
-        # Otherwise, show separately
-        return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show]
-
-    sensor_ids_to_show = asset.sensors_to_show
-    # Import the schema for validation
-    from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema
-
-    sensors_to_show_schema = SensorsToShowSchema()
-
-    # Deserialize the sensor_ids_to_show using SensorsToShowSchema
-    standardized_sensors_to_show = sensors_to_show_schema.deserialize(
-        sensor_ids_to_show
-    )
-
-    sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show)
-
-    # Only allow showing sensors from assets owned by the user's organization,
-    # except in play mode, where any sensor may be shown
-    accounts = [asset.owner] if asset.owner is not None else None
-    if current_app.config.get("FLEXMEASURES_MODE") == "play":
-        from flexmeasures.data.models.user import Account
-
-        accounts = db.session.scalars(select(Account)).all()
-
-    from flexmeasures.data.services.sensors import get_sensors
-
-    accessible_sensor_map = {
-        sensor.id: sensor
-        for sensor in get_sensors(
-            account=accounts,
-            include_public_assets=True,
-            sensor_id_allowlist=sensor_id_allowlist,
-        )
-    }
-
-    # Build list of sensor objects that are accessible
-    sensors_to_show = []
-    missed_sensor_ids = []
-
-    for entry in standardized_sensors_to_show:
-
-        title = entry.get("title")
-        sensors = entry.get("sensors")
-
-        accessible_sensors = [
-            accessible_sensor_map.get(sid)
-            for sid in sensors
-            if sid in accessible_sensor_map
-        ]
-        inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map]
-        missed_sensor_ids.extend(inaccessible)
-        if accessible_sensors:
-            sensors_to_show.append({"title": title, "sensors": accessible_sensors})
-
-    if missed_sensor_ids:
-        current_app.logger.warning(
-            f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {asset.id}, as it is not accessible to user {current_user}."
-        )
-    # check if sensors_to_show was updated
-    if old_sensors_to_show != asset.sensors_to_show:
-        asset.attributes["sensors_to_show"] = standardized_sensors_to_show
-    db.session.commit()
-
-    return sensors_to_show