Sensorstoshow table column

documentation/changelog.rst

@@ -32,6 +32,7 @@ New features
 -------------
 * New chart type on sensor page: histogram [see `PR #1143 <https://github.com/FlexMeasures/flexmeasures/pull/1143>`_]
 * Add basic sensor info to sensor page [see `PR #1115 <https://github.com/FlexMeasures/flexmeasures/pull/1115>`_]
+* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 <https://github.com/FlexMeasures/flexmeasures/pull/1200>`_]
 * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors/<id>/stats` endpoint to get sensor statistics [see `PR #1116 <https://github.com/FlexMeasures/flexmeasures/pull/1116>`_]
 * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 <https://github.com/FlexMeasures/flexmeasures/pull/1125>`_ and `PR #1177 <https://github.com/FlexMeasures/flexmeasures/pull/1177>`_]
 * Support zoom-in action on the asset and sensor charts [see `PR #1130 <https://github.com/FlexMeasures/flexmeasures/pull/1130>`_]

flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py

@@ -0,0 +1,61 @@
+"""add sensors_to_show field in asset model
+
+Revision ID: 950e23e3aa54
+Revises: 0af134879301
+Create Date: 2024-09-27 10:21:37.910186
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import select
+
+# revision identifiers, used by Alembic.
+revision = "950e23e3aa54"
+down_revision = "0af134879301"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add the 'sensors_to_show' column with nullable=True since we will populate it
+    with op.batch_alter_table("generic_asset", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("sensors_to_show", sa.JSON(), nullable=True))
+
+    generic_asset_table = sa.Table(
+        "generic_asset",
+        sa.MetaData(),
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("attributes", sa.JSON),
+        sa.Column("sensors_to_show", sa.JSON),
+    )
+
+    # Initiate connection to execute the queries
+    conn = op.get_bind()
+
+    select_stmt = select(generic_asset_table.c.id, generic_asset_table.c.attributes)
+    results = conn.execute(select_stmt)
+
+    for row in results:
+        asset_id, attributes_data = row
+
+        sensors_to_show = attributes_data.get("sensors_to_show", [])
+
+        if not isinstance(sensors_to_show, list):
+            sensors_to_show = [sensors_to_show]
+
+        update_stmt = (
+            generic_asset_table.update()
+            .where(generic_asset_table.c.id == asset_id)
+            .values(sensors_to_show=sensors_to_show)
+        )
+        conn.execute(update_stmt)
+
+    # After populating column, set back to be NOT NULL
+    with op.batch_alter_table("generic_asset", schema=None) as batch_op:
+        batch_op.alter_column("sensors_to_show", nullable=False)
+
+
+def downgrade():
+    with op.batch_alter_table("generic_asset", schema=None) as batch_op:
+        batch_op.drop_column("sensors_to_show")

flexmeasures/data/models/generic_assets.py

@@ -4,14 +4,13 @@
 from typing import Any
 import json
 
-from flask import current_app
 from flask_security import current_user
 import pandas as pd
 from sqlalchemy import select
 from sqlalchemy.engine import Row
 from sqlalchemy.ext.hybrid import hybrid_method
 from sqlalchemy.sql.expression import func, text
-from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.ext.mutable import MutableDict, MutableList
 from timely_beliefs import BeliefsDataFrame, utils as tb_utils
 
 from flexmeasures.data import db
@@ -24,7 +23,7 @@
 from flexmeasures.data.services.timerange import get_timerange
 from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER
 from flexmeasures.utils import geo_utils
-from flexmeasures.utils.coding_utils import flatten_unique
+from flexmeasures.utils.coding_utils import flatten_unique, process_sensors
 from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution
 
 
@@ -86,6 +85,9 @@ class GenericAsset(db.Model, AuthModelMixin):
     latitude = db.Column(db.Float, nullable=True)
     longitude = db.Column(db.Float, nullable=True)
     attributes = db.Column(MutableDict.as_mutable(db.JSON), nullable=False, default={})
+    sensors_to_show = db.Column(
+        MutableList.as_mutable(db.JSON), nullable=False, default=[]
+    )
 
     # One-to-many (or many-to-one?) relationships
     parent_asset_id = db.Column(
@@ -139,6 +141,10 @@ class GenericAsset(db.Model, AuthModelMixin):
         ),
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        process_sensors(self)  # Migrate data to sensors_to_show
+
     def __acl__(self):
         """
         All logged-in users can read if the asset is public.
@@ -452,7 +458,8 @@ def chart(
         :param resolution: optionally set the resolution of data being displayed
         :returns: JSON string defining vega-lite chart specs
         """
-        sensors = flatten_unique(self.sensors_to_show)
+        processed_sensors_to_show = process_sensors(self)
+        sensors = flatten_unique(processed_sensors_to_show)
         for sensor in sensors:
             sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name)
 
@@ -465,7 +472,7 @@ def chart(
             kwargs["event_ends_before"] = event_ends_before
         chart_specs = chart_type_to_chart_specs(
             chart_type,
-            sensors_to_show=self.sensors_to_show,
+            sensors_to_show=processed_sensors_to_show,
             dataset_name=dataset_name,
             combine_legend=combine_legend,
             **kwargs,
@@ -598,120 +605,6 @@ def search_beliefs(
             return df.to_json(orient="records")
         return bdf_dict
 
-    @property
-    def sensors_to_show(
-        self,
-    ) -> list[dict[str, "Sensor"]]:  # noqa F821
-        """
-        Sensors to show, as defined by the sensors_to_show attribute.
-
-        Sensors to show are defined as a list of sensor IDs, which are set by the "sensors_to_show" field in the asset's "attributes" column.
-        Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets.
-        In play mode, sensors from different accounts can be added.
-
-        Sensor IDs can be nested to denote that sensors should be 'shown together', for example, layered rather than vertically concatenated.
-        Additionally, each row of sensors can be accompanied by a title.
-        If no title is provided, `"title": None` will be assigned in the returned dictionary.
-
-        How to interpret 'shown together' is technically left up to the function returning chart specifications, as are any restrictions regarding which sensors can be shown together, such as:
-        - Whether they should share the same unit
-        - Whether they should share the same name
-        - Whether they should belong to different assets
-
-        For example, this input denotes showing sensors 42 and 44 together:
-
-            sensors_to_show = [40, 35, 41, [42, 44], 43, 45]
-
-        And this input denotes showing sensors 42 and 44 together with a custom title:
-
-            sensors_to_show = [
-                {"title": "Title 1", "sensor": 40},
-                {"title": "Title 2", "sensors": [41, 42]},
-                [43, 44], 45, 46
-            ]
-
-        In both cases, the returned format will contain sensor objects mapped to their respective sensor IDs, as follows:
-
-            [
-                {"title": "Title 1", "sensor": <Sensor object for sensor 40>},
-                {"title": "Title 2", "sensors": [<Sensor object for sensor 41>, <Sensor object for sensor 42>]},
-                {"title": None, "sensors": [<Sensor object for sensor 43>, <Sensor object for sensor 44>]},
-                {"title": None, "sensor": <Sensor object for sensor 45>},
-                {"title": None, "sensor": <Sensor object for sensor 46>}
-            ]
-
-        In case the `sensors_to_show` field is missing, it defaults to two of the asset's sensors. These will be shown together (e.g., sharing the same y-axis) if they share the same unit; otherwise, they will be shown separately.
-
-        Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view.
-        """
-        if not self.has_attribute("sensors_to_show"):
-            sensors_to_show = self.sensors[:2]
-            if (
-                len(sensors_to_show) == 2
-                and sensors_to_show[0].unit == sensors_to_show[1].unit
-            ):
-                # Sensors are shown together (e.g. they can share the same y-axis)
-                return [{"title": None, "sensors": sensors_to_show}]
-            # Otherwise, show separately
-            return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show]
-
-        sensor_ids_to_show = self.get_attribute("sensors_to_show")
-        # Import the schema for validation
-        from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema
-
-        sensors_to_show_schema = SensorsToShowSchema()
-
-        # Deserialize the sensor_ids_to_show using SensorsToShowSchema
-        standardized_sensors_to_show = sensors_to_show_schema.deserialize(
-            sensor_ids_to_show
-        )
-
-        sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show)
-
-        # Only allow showing sensors from assets owned by the user's organization,
-        # except in play mode, where any sensor may be shown
-        accounts = [self.owner] if self.owner is not None else None
-        if current_app.config.get("FLEXMEASURES_MODE") == "play":
-            from flexmeasures.data.models.user import Account
-
-            accounts = db.session.scalars(select(Account)).all()
-
-        from flexmeasures.data.services.sensors import get_sensors
-
-        accessible_sensor_map = {
-            sensor.id: sensor
-            for sensor in get_sensors(
-                account=accounts,
-                include_public_assets=True,
-                sensor_id_allowlist=sensor_id_allowlist,
-            )
-        }
-
-        # Build list of sensor objects that are accessible
-        sensors_to_show = []
-        missed_sensor_ids = []
-
-        for entry in standardized_sensors_to_show:
-
-            title = entry.get("title")
-            sensors = entry.get("sensors")
-
-            accessible_sensors = [
-                accessible_sensor_map.get(sid)
-                for sid in sensors
-                if sid in accessible_sensor_map
-            ]
-            inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map]
-            missed_sensor_ids.extend(inaccessible)
-            if accessible_sensors:
-                sensors_to_show.append({"title": title, "sensors": accessible_sensors})
-
-        if missed_sensor_ids:
-            current_app.logger.warning(
-                f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}."
-            )
-        return sensors_to_show
-
     @property
     def timezone(
         self,
@@ -748,7 +641,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]:
                       'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
                   }
         """
-        return self.get_timerange(self.sensors_to_show)
+        return self.get_timerange(process_sensors(self))
 
     @classmethod
     def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]:  # noqa F821

flexmeasures/data/schemas/generic_assets.py

@@ -6,6 +6,7 @@
 from flask_security import current_user
 from sqlalchemy import select
 
+
 from flexmeasures.data import ma, db
 from flexmeasures.data.models.user import Account
 from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
@@ -172,6 +173,7 @@ class GenericAssetSchema(ma.SQLAlchemySchema):
         only=("id", "name", "account_id", "generic_asset_type"),
     )
     sensors = ma.Nested("SensorSchema", many=True, dump_only=True, only=("id", "name"))
+    sensors_to_show = JSON(required=False)
     production_price_sensor_id = fields.Int(required=False, allow_none=True)
     consumption_price_sensor_id = fields.Int(required=False, allow_none=True)
     inflexible_device_sensor_ids = fields.List(
@@ -238,7 +240,6 @@ def validate_account(self, account_id: int | None):
     @validates("attributes")
     def validate_attributes(self, attributes: dict):
         sensors_to_show = attributes.get("sensors_to_show", [])
-
         if sensors_to_show:
 
             # Use SensorsToShowSchema to validate and deserialize sensors_to_show