From 2cc92fbfb1e077adea7de7d3adb9a82353ec72f8 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 27 Sep 2024 09:58:04 +0100 Subject: [PATCH 01/24] chore: added sensors_to_show field in asset model Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 74b001079..cd3be4378 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -86,6 +86,9 @@ class GenericAsset(db.Model, AuthModelMixin): latitude = db.Column(db.Float, nullable=True) longitude = db.Column(db.Float, nullable=True) attributes = db.Column(MutableDict.as_mutable(db.JSON), nullable=False, default={}) + sensors_to_show = db.Column( + MutableDict.as_mutable(db.JSON), nullable=False, default={} + ) # One-to-many (or many-to-one?) relationships parent_asset_id = db.Column( @@ -599,7 +602,7 @@ def search_beliefs( return bdf_dict @property - def sensors_to_show( + def sensors_to_show_func( self, ) -> list[dict[str, "Sensor"]]: # noqa F821 """ From c914e5580edbef5c0be6fba3e37ec06d9d9b086d Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 27 Sep 2024 12:13:22 +0100 Subject: [PATCH 02/24] chore: sensors_toShow migration Signed-off-by: joshuaunity --- ...dd_sensors_to_show_field_in_asset_model.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py diff --git a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py new file mode 100644 index 000000000..06ad525b4 --- /dev/null +++ b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py @@ -0,0 +1,58 @@ +"""add sensors_to_show field in asset model + +Revision ID: 950e23e3aa54 +Revises: 0af134879301 +Create Date: 2024-09-27 10:21:37.910186 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import select + +# revision identifiers, used by Alembic. +revision = "950e23e3aa54" +down_revision = "0af134879301" +branch_labels = None +depends_on = None + + +def upgrade(): + # Add the 'sensors_to_show' column with nullable=True since we will populate it + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.add_column(sa.Column("sensors_to_show", sa.JSON(), nullable=True)) + + generic_asset_table = sa.Table( + "generic_asset", + sa.MetaData(), + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("attributes", sa.JSON), + sa.Column("sensors_to_show", sa.JSON), + ) + + # Initiate connection to execute the queries + conn = op.get_bind() + + select_stmt = select(generic_asset_table.c.id, generic_asset_table.c.attributes) + results = conn.execute(select_stmt) + + for row in results: + asset_id, attributes_data = row + + sensors_to_show = attributes_data.get("sensors_to_show", {}) + + update_stmt = ( + generic_asset_table.update() + .where(generic_asset_table.c.id == asset_id) + .values(sensors_to_show=sensors_to_show) + ) + conn.execute(update_stmt) + + # After populating column, set back to be NOT NULL + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.alter_column("sensors_to_show", nullable=False) + + +def downgrade(): + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.drop_column("sensors_to_show") From 7ac5c6c0c114ad2b27110adbd96fdec59b58a981 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 30 Sep 2024 13:15:40 +0100 Subject: [PATCH 03/24] chore: sensors to show logic in progress Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/assets.py | 3 + ...dd_sensors_to_show_field_in_asset_model.py | 2 +- flexmeasures/data/models/generic_assets.py | 127 ++---------------- flexmeasures/data/schemas/generic_assets.py | 3 +- flexmeasures/utils/coding_utils.py | 118 ++++++++++++++++ 5 files changed, 136 insertions(+), 117 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 54301d029..d46c2af5f 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -169,6 +169,9 @@ def index( search_terms=filter, filter_statement=filter_statement ) if page is None: + assets = db.session.scalars(query).all() + print("====================", assets[0].sensors_to_show) + print("====================", assets[0]) response = asset_schema.dump(db.session.scalars(query).all(), many=True) else: if per_page is None: diff --git a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py index 06ad525b4..b729900e5 100644 --- a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py +++ b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py @@ -39,7 +39,7 @@ def upgrade(): for row in results: asset_id, attributes_data = row - sensors_to_show = attributes_data.get("sensors_to_show", {}) + sensors_to_show = attributes_data.get("sensors_to_show", []) update_stmt = ( generic_asset_table.update() diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index cd3be4378..222587306 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -4,7 +4,6 @@ from typing import Any import json -from flask import current_app from flask_security import current_user import pandas as pd from sqlalchemy import select @@ -142,6 +141,18 @@ class GenericAsset(db.Model, AuthModelMixin): ), ) + # def __init__( + # self, name, latitude=None, longitude=None, attributes=None, sensors=None + # ): + # super().__init__( + # name=name, latitude=latitude, longitude=longitude, attributes=attributes + # ) + + # # Process the sensors data + # processed_sensors_to_show = process_sensors(self) + # print("====================: ", {"sensors_to_show": processed_sensors_to_show}) + # self.sensors_to_show = {"sensors_to_show": processed_sensors_to_show} + def __acl__(self): """ All logged-in users can read if the asset is public. @@ -601,120 +612,6 @@ def search_beliefs( return df.to_json(orient="records") return bdf_dict - @property - def sensors_to_show_func( - self, - ) -> list[dict[str, "Sensor"]]: # noqa F821 - """ - Sensors to show, as defined by the sensors_to_show attribute. - - Sensors to show are defined as a list of sensor IDs, which are set by the "sensors_to_show" field in the asset's "attributes" column. - Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets. - In play mode, sensors from different accounts can be added. - - Sensor IDs can be nested to denote that sensors should be 'shown together', for example, layered rather than vertically concatenated. - Additionally, each row of sensors can be accompanied by a title. - If no title is provided, `"title": None` will be assigned in the returned dictionary. - - How to interpret 'shown together' is technically left up to the function returning chart specifications, as are any restrictions regarding which sensors can be shown together, such as: - - Whether they should share the same unit - - Whether they should share the same name - - Whether they should belong to different assets - - For example, this input denotes showing sensors 42 and 44 together: - - sensors_to_show = [40, 35, 41, [42, 44], 43, 45] - - And this input denotes showing sensors 42 and 44 together with a custom title: - - sensors_to_show = [ - {"title": "Title 1", "sensor": 40}, - {"title": "Title 2", "sensors": [41, 42]}, - [43, 44], 45, 46 - ] - - In both cases, the returned format will contain sensor objects mapped to their respective sensor IDs, as follows: - - [ - {"title": "Title 1", "sensor": }, - {"title": "Title 2", "sensors": [, ]}, - {"title": None, "sensors": [, ]}, - {"title": None, "sensor": }, - {"title": None, "sensor": } - ] - - In case the `sensors_to_show` field is missing, it defaults to two of the asset's sensors. These will be shown together (e.g., sharing the same y-axis) if they share the same unit; otherwise, they will be shown separately. - - Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view. - """ - if not self.has_attribute("sensors_to_show"): - sensors_to_show = self.sensors[:2] - if ( - len(sensors_to_show) == 2 - and sensors_to_show[0].unit == sensors_to_show[1].unit - ): - # Sensors are shown together (e.g. they can share the same y-axis) - return [{"title": None, "sensors": sensors_to_show}] - # Otherwise, show separately - return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] - - sensor_ids_to_show = self.get_attribute("sensors_to_show") - # Import the schema for validation - from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema - - sensors_to_show_schema = SensorsToShowSchema() - - # Deserialize the sensor_ids_to_show using SensorsToShowSchema - standardized_sensors_to_show = sensors_to_show_schema.deserialize( - sensor_ids_to_show - ) - - sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) - - # Only allow showing sensors from assets owned by the user's organization, - # except in play mode, where any sensor may be shown - accounts = [self.owner] if self.owner is not None else None - if current_app.config.get("FLEXMEASURES_MODE") == "play": - from flexmeasures.data.models.user import Account - - accounts = db.session.scalars(select(Account)).all() - - from flexmeasures.data.services.sensors import get_sensors - - accessible_sensor_map = { - sensor.id: sensor - for sensor in get_sensors( - account=accounts, - include_public_assets=True, - sensor_id_allowlist=sensor_id_allowlist, - ) - } - - # Build list of sensor objects that are accessible - sensors_to_show = [] - missed_sensor_ids = [] - - for entry in standardized_sensors_to_show: - - title = entry.get("title") - sensors = entry.get("sensors") - - accessible_sensors = [ - accessible_sensor_map.get(sid) - for sid in sensors - if sid in accessible_sensor_map - ] - inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] - missed_sensor_ids.extend(inaccessible) - if accessible_sensors: - sensors_to_show.append({"title": title, "sensors": accessible_sensors}) - - if missed_sensor_ids: - current_app.logger.warning( - f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}." - ) - return sensors_to_show - @property def timezone( self, diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index fba44fe2b..c1b739c6c 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -6,6 +6,7 @@ from flask_security import current_user from sqlalchemy import select + from flexmeasures.data import ma, db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @@ -172,6 +173,7 @@ class GenericAssetSchema(ma.SQLAlchemySchema): only=("id", "name", "account_id", "generic_asset_type"), ) sensors = ma.Nested("SensorSchema", many=True, dump_only=True, only=("id", "name")) + sensors_to_show = SensorsToShowSchema(required=False) production_price_sensor_id = fields.Int(required=False, allow_none=True) consumption_price_sensor_id = fields.Int(required=False, allow_none=True) inflexible_device_sensor_ids = fields.List( @@ -238,7 +240,6 @@ def validate_account(self, account_id: int | None): @validates("attributes") def validate_attributes(self, attributes: dict): sensors_to_show = attributes.get("sensors_to_show", []) - if sensors_to_show: # Use SensorsToShowSchema to validate and deserialize sensors_to_show diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index d5f26a4a3..4ad7d9d36 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -7,7 +7,13 @@ import inspect import importlib import pkgutil + +from sqlalchemy import select from flask import current_app +from flask_security import current_user + +from flexmeasures.data import db +from flexmeasures.data.models.time_series import Sensor def delete_key_recursive(value, key): @@ -176,3 +182,115 @@ def find_classes_modules(module, superclass, skiptest=True): def get_classes_module(module, superclass, skiptest=True) -> dict: return dict(find_classes_modules(module, superclass, skiptest=skiptest)) + + +def process_sensors(asset) -> list[dict[str, "Sensor"]]: + """ + Sensors to show, as defined by the sensors_to_show attribute. + + Sensors to show are defined as a list of sensor IDs, which are set by the "sensors_to_show" field in the asset's "attributes" column. + Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets. + In play mode, sensors from different accounts can be added. + + Sensor IDs can be nested to denote that sensors should be 'shown together', for example, layered rather than vertically concatenated. + Additionally, each row of sensors can be accompanied by a title. + If no title is provided, `"title": None` will be assigned in the returned dictionary. + + How to interpret 'shown together' is technically left up to the function returning chart specifications, as are any restrictions regarding which sensors can be shown together, such as: + - Whether they should share the same unit + - Whether they should share the same name + - Whether they should belong to different assets + + For example, this input denotes showing sensors 42 and 44 together: + + sensors_to_show = [40, 35, 41, [42, 44], 43, 45] + + And this input denotes showing sensors 42 and 44 together with a custom title: + + sensors_to_show = [ + {"title": "Title 1", "sensor": 40}, + {"title": "Title 2", "sensors": [41, 42]}, + [43, 44], 45, 46 + ] + + In both cases, the returned format will contain sensor objects mapped to their respective sensor IDs, as follows: + + [ + {"title": "Title 1", "sensor": }, + {"title": "Title 2", "sensors": [, ]}, + {"title": None, "sensors": [, ]}, + {"title": None, "sensor": }, + {"title": None, "sensor": } + ] + + In case the `sensors_to_show` field is missing, it defaults to two of the asset's sensors. These will be shown together (e.g., sharing the same y-axis) if they share the same unit; otherwise, they will be shown separately. + + Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view. + """ + if not asset.sensors_to_show or asset.sensors_to_show == {}: + sensors_to_show = asset.sensors[:2] + if ( + len(sensors_to_show) == 2 + and sensors_to_show[0].unit == sensors_to_show[1].unit + ): + # Sensors are shown together (e.g. they can share the same y-axis) + return [{"title": None, "sensors": sensors_to_show}] + # Otherwise, show separately + return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] + + sensor_ids_to_show = asset.sensors_to_show + # Import the schema for validation + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + + sensors_to_show_schema = SensorsToShowSchema() + + # Deserialize the sensor_ids_to_show using SensorsToShowSchema + standardized_sensors_to_show = sensors_to_show_schema.deserialize( + sensor_ids_to_show + ) + + sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) + + # Only allow showing sensors from assets owned by the user's organization, + # except in play mode, where any sensor may be shown + accounts = [asset.owner] if asset.owner is not None else None + if current_app.config.get("FLEXMEASURES_MODE") == "play": + from flexmeasures.data.models.user import Account + + accounts = db.session.scalars(select(Account)).all() + + from flexmeasures.data.services.sensors import get_sensors + + accessible_sensor_map = { + sensor.id: sensor + for sensor in get_sensors( + account=accounts, + include_public_assets=True, + sensor_id_allowlist=sensor_id_allowlist, + ) + } + + # Build list of sensor objects that are accessible + sensors_to_show = [] + missed_sensor_ids = [] + + for entry in standardized_sensors_to_show: + + title = entry.get("title") + sensors = entry.get("sensors") + + accessible_sensors = [ + accessible_sensor_map.get(sid) + for sid in sensors + if sid in accessible_sensor_map + ] + inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] + missed_sensor_ids.extend(inaccessible) + if accessible_sensors: + sensors_to_show.append({"title": title, "sensors": accessible_sensors}) + + if missed_sensor_ids: + current_app.logger.warning( + f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {asset.id}, as it is not accessible to user {current_user}." + ) + return sensors_to_show From 16e99d6d9b87f362ae49cb6fbc1ff58eba949053 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 30 Sep 2024 16:14:59 +0100 Subject: [PATCH 04/24] fix: sensor_to_show field unreadable Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/assets.py | 3 --- ...dd_sensors_to_show_field_in_asset_model.py | 3 +++ flexmeasures/data/models/generic_assets.py | 21 +++++++------------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index d46c2af5f..54301d029 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -169,9 +169,6 @@ def index( search_terms=filter, filter_statement=filter_statement ) if page is None: - assets = db.session.scalars(query).all() - print("====================", assets[0].sensors_to_show) - print("====================", assets[0]) response = asset_schema.dump(db.session.scalars(query).all(), many=True) else: if per_page is None: diff --git a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py index b729900e5..fccf85484 100644 --- a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py +++ b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py @@ -41,6 +41,9 @@ def upgrade(): sensors_to_show = attributes_data.get("sensors_to_show", []) + if not isinstance(sensors_to_show, list): + sensors_to_show = [sensors_to_show] + update_stmt = ( generic_asset_table.update() .where(generic_asset_table.c.id == asset_id) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 222587306..76e64656c 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -10,7 +10,7 @@ from sqlalchemy.engine import Row from sqlalchemy.ext.hybrid import hybrid_method from sqlalchemy.sql.expression import func, text -from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.ext.mutable import MutableDict, MutableList from timely_beliefs import BeliefsDataFrame, utils as tb_utils from flexmeasures.data import db @@ -23,7 +23,7 @@ from flexmeasures.data.services.timerange import get_timerange from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.utils import geo_utils -from flexmeasures.utils.coding_utils import flatten_unique +from flexmeasures.utils.coding_utils import flatten_unique, process_sensors from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution @@ -86,7 +86,7 @@ class GenericAsset(db.Model, AuthModelMixin): longitude = db.Column(db.Float, nullable=True) attributes = db.Column(MutableDict.as_mutable(db.JSON), nullable=False, default={}) sensors_to_show = db.Column( - MutableDict.as_mutable(db.JSON), nullable=False, default={} + MutableList.as_mutable(db.JSON), nullable=False, default=[] ) # One-to-many (or many-to-one?) relationships @@ -141,17 +141,10 @@ class GenericAsset(db.Model, AuthModelMixin): ), ) - # def __init__( - # self, name, latitude=None, longitude=None, attributes=None, sensors=None - # ): - # super().__init__( - # name=name, latitude=latitude, longitude=longitude, attributes=attributes - # ) - - # # Process the sensors data - # processed_sensors_to_show = process_sensors(self) - # print("====================: ", {"sensors_to_show": processed_sensors_to_show}) - # self.sensors_to_show = {"sensors_to_show": processed_sensors_to_show} + def __init__(self, *args, **kwargs): + processed_sensors_to_show = process_sensors(self) + self.sensors_to_show = processed_sensors_to_show + super().__init__(*args, **kwargs) def __acl__(self): """ From 5bfbc09645665f525f2ab66de18878e9c07bc422 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 30 Sep 2024 16:27:59 +0100 Subject: [PATCH 05/24] fix: import error due absence of linting supresser noqa F821 Signed-off-by: joshuaunity --- flexmeasures/utils/coding_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 4ad7d9d36..a2ebadba9 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -13,7 +13,6 @@ from flask_security import current_user from flexmeasures.data import db -from flexmeasures.data.models.time_series import Sensor def delete_key_recursive(value, key): @@ -184,7 +183,7 @@ def get_classes_module(module, superclass, skiptest=True) -> dict: return dict(find_classes_modules(module, superclass, skiptest=skiptest)) -def process_sensors(asset) -> list[dict[str, "Sensor"]]: +def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 """ Sensors to show, as defined by the sensors_to_show attribute. From 73f3b61ed0490606e824ac3bc99507444517328a Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 1 Oct 2024 11:40:34 +0100 Subject: [PATCH 06/24] feat: auto migrate sensors_to_show data Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 10 +++++----- flexmeasures/utils/coding_utils.py | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 76e64656c..5ffb87601 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -142,9 +142,8 @@ class GenericAsset(db.Model, AuthModelMixin): ) def __init__(self, *args, **kwargs): - processed_sensors_to_show = process_sensors(self) - self.sensors_to_show = processed_sensors_to_show super().__init__(*args, **kwargs) + process_sensors(self) # Migrate data to sensors_to_show def __acl__(self): """ @@ -459,7 +458,8 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ - sensors = flatten_unique(self.sensors_to_show) + processed_sensors_to_show = process_sensors(self) + sensors = flatten_unique(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -472,7 +472,7 @@ def chart( kwargs["event_ends_before"] = event_ends_before chart_specs = chart_type_to_chart_specs( chart_type, - sensors_to_show=self.sensors_to_show, + sensors_to_show=processed_sensors_to_show, dataset_name=dataset_name, combine_legend=combine_legend, **kwargs, @@ -641,7 +641,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]: 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - return self.get_timerange(self.sensors_to_show) + return self.get_timerange(process_sensors(self)) @classmethod def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa F821 diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index a2ebadba9..9c5e85681 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -226,6 +226,10 @@ def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view. """ + old_sensors_to_show = ( + asset.sensors_to_show + ) # Used to check if sensors_to_show was updated + asset.sensors_to_show = asset.attributes.get("sensors_to_show", []) if not asset.sensors_to_show or asset.sensors_to_show == {}: sensors_to_show = asset.sensors[:2] if ( @@ -292,4 +296,9 @@ def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 current_app.logger.warning( f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {asset.id}, as it is not accessible to user {current_user}." ) + # check if sensors_to_show was updated + if old_sensors_to_show != asset.sensors_to_show: + asset.attributes["sensors_to_show"] = standardized_sensors_to_show + db.session.commit() + return sensors_to_show From 0191b79972eaa03d903b84c43f37960f3d6b3cfa Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 2 Oct 2024 10:17:18 +0100 Subject: [PATCH 07/24] refactor: handle empty case Signed-off-by: joshuaunity --- flexmeasures/utils/coding_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 9c5e85681..bc678d080 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -229,7 +229,11 @@ def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 old_sensors_to_show = ( asset.sensors_to_show ) # Used to check if sensors_to_show was updated - asset.sensors_to_show = asset.attributes.get("sensors_to_show", []) + if asset.attributes is not None: + asset.sensors_to_show = asset.attributes.get("sensors_to_show", []) + else: + asset.sensors_to_show = [] + if not asset.sensors_to_show or asset.sensors_to_show == {}: sensors_to_show = asset.sensors[:2] if ( From a44926a7e16bc92011a4def2b7f027c2d57ba117 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 2 Oct 2024 11:19:27 +0100 Subject: [PATCH 08/24] fix: used proper field type for sensors_to_sow Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index c1b739c6c..3658e082a 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -173,7 +173,7 @@ class GenericAssetSchema(ma.SQLAlchemySchema): only=("id", "name", "account_id", "generic_asset_type"), ) sensors = ma.Nested("SensorSchema", many=True, dump_only=True, only=("id", "name")) - sensors_to_show = SensorsToShowSchema(required=False) + sensors_to_show = JSON(required=False) production_price_sensor_id = fields.Int(required=False, allow_none=True) consumption_price_sensor_id = fields.Int(required=False, allow_none=True) inflexible_device_sensor_ids = fields.List( From 2bd0d7ab920f0a6afa749f07f73a4c64560bb15d Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 2 Oct 2024 11:53:47 +0100 Subject: [PATCH 09/24] chore: added to changelog Signed-off-by: joshuaunity --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 50cb2fe50..8b8b1be29 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -32,6 +32,7 @@ New features ------------- * New chart type on sensor page: histogram [see `PR #1143 `_] * Add basic sensor info to sensor page [see `PR #1115 `_] +* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 `] * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors//stats` endpoint to get sensor statistics [see `PR #1116 `_] * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 `_ and `PR #1177 `_] * Support zoom-in action on the asset and sensor charts [see `PR #1130 `_] From e9d2cfa638026183f8f2e8ad52d928f09746ede7 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 2 Oct 2024 12:03:57 +0100 Subject: [PATCH 10/24] chore: typo fix Signed-off-by: joshuaunity --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 8b8b1be29..9f9019dd3 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -32,7 +32,7 @@ New features ------------- * New chart type on sensor page: histogram [see `PR #1143 `_] * Add basic sensor info to sensor page [see `PR #1115 `_] -* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 `] +* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 `_] * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors//stats` endpoint to get sensor statistics [see `PR #1116 `_] * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 `_ and `PR #1177 `_] * Support zoom-in action on the asset and sensor charts [see `PR #1130 `_] From 628dd98e535717b5508a5ad72fdb46ccef721a99 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 3 Oct 2024 10:22:57 +0100 Subject: [PATCH 11/24] chore: updated changelog Signed-off-by: joshuaunity --- documentation/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9f9019dd3..8dc2da4a5 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -28,11 +28,13 @@ v0.23.0 | September 18, 2024 .. note:: Read more on these features on `the FlexMeasures blog `_. +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + New features ------------- * New chart type on sensor page: histogram [see `PR #1143 `_] * Add basic sensor info to sensor page [see `PR #1115 `_] -* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data fro parent source(attributes field) [see `PR #1200 `_] +* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data from parent source(attributes field) [see `PR #1200 `_] * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors//stats` endpoint to get sensor statistics [see `PR #1116 `_] * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 `_ and `PR #1177 `_] * Support zoom-in action on the asset and sensor charts [see `PR #1130 `_] From 1dfb9468c1de994e4bdcb34c1b79e854861f5d5b Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 3 Oct 2024 11:06:15 +0100 Subject: [PATCH 12/24] chore: func name change Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 8 ++++---- flexmeasures/utils/coding_utils.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 5ffb87601..a13eb3650 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -23,7 +23,7 @@ from flexmeasures.data.services.timerange import get_timerange from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.utils import geo_utils -from flexmeasures.utils.coding_utils import flatten_unique, process_sensors +from flexmeasures.utils.coding_utils import flatten_unique, validate_sesnors_to_show from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution @@ -143,7 +143,7 @@ class GenericAsset(db.Model, AuthModelMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - process_sensors(self) # Migrate data to sensors_to_show + validate_sesnors_to_show(self) # Migrate data to sensors_to_show def __acl__(self): """ @@ -458,7 +458,7 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ - processed_sensors_to_show = process_sensors(self) + processed_sensors_to_show = validate_sesnors_to_show(self) sensors = flatten_unique(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -641,7 +641,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]: 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - return self.get_timerange(process_sensors(self)) + return self.get_timerange(validate_sesnors_to_show(self)) @classmethod def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa F821 diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index bc678d080..7c520346a 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -183,7 +183,9 @@ def get_classes_module(module, superclass, skiptest=True) -> dict: return dict(find_classes_modules(module, superclass, skiptest=skiptest)) -def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 +def validate_sesnors_to_show( + asset, +) -> list[dict[str, str | None | "Sensor" | list["Sensor"]]]: # noqa: F821 """ Sensors to show, as defined by the sensors_to_show attribute. @@ -234,7 +236,7 @@ def process_sensors(asset) -> list[dict[str, "Sensor"]]: # noqa F821 else: asset.sensors_to_show = [] - if not asset.sensors_to_show or asset.sensors_to_show == {}: + if not asset.sensors_to_show or asset.sensors_to_show == []: sensors_to_show = asset.sensors[:2] if ( len(sensors_to_show) == 2 From ac64344ad463fe63564ecfeda65fb5ee50600791 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 3 Oct 2024 12:06:51 +0100 Subject: [PATCH 13/24] refactor: updated doc string and fixed does not accept objects of type err Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 129 +++++++++++++++++++- flexmeasures/ui/crud/assets/views.py | 9 ++ flexmeasures/utils/coding_utils.py | 131 --------------------- 3 files changed, 134 insertions(+), 135 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index a13eb3650..0426d11de 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -4,6 +4,7 @@ from typing import Any import json +from flask import current_app from flask_security import current_user import pandas as pd from sqlalchemy import select @@ -23,7 +24,7 @@ from flexmeasures.data.services.timerange import get_timerange from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.utils import geo_utils -from flexmeasures.utils.coding_utils import flatten_unique, validate_sesnors_to_show +from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution @@ -143,7 +144,6 @@ class GenericAsset(db.Model, AuthModelMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - validate_sesnors_to_show(self) # Migrate data to sensors_to_show def __acl__(self): """ @@ -170,6 +170,127 @@ def __repr__(self): self.generic_asset_type.name, ) + def validate_sensors_to_show( + self, + ) -> list[dict[str, str | None | "Sensor" | list["Sensor"]]]: # noqa: F821 + """ + Validate and transform the 'sensors_to_show' attribute into the latest format for use in graph-making code. + + This function ensures that the 'sensors_to_show' attribute: + 1. Follows the latest format, even if the data in the database uses an older format. + 2. Contains only sensors that the user has access to (based on the current asset, account, or public availability). + 3. Returns a list of dictionaries where each dictionary contains either a single sensor or a group of sensors with an optional title. + + Steps: + - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly. + - It checks if each sensor is accessible by the user and filters out any unauthorized sensors. + - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles. + + Details on format: + - The 'sensors_to_show' attribute is defined as a list of sensor IDs or nested lists of sensor IDs (to indicate grouping). + - Titles may be associated with rows of sensors. If no title is provided, `{"title": None}` will be assigned. + - Nested lists of sensors indicate that they should be shown together (e.g., layered in the same chart). + + Example inputs: + 1. Simple list of sensors, where sensors 42 and 44 are grouped: + sensors_to_show = [40, 35, 41, [42, 44], 43, 45] + + 2. List with titles and sensor groupings: + sensors_to_show = [ + {"title": "Title 1", "sensor": 40}, + {"title": "Title 2", "sensors": [41, 42]}, + [43, 44], 45, 46 + ] + + Returned structure: + - The function returns a list of dictionaries, with each dictionary containing either a 'sensor' (for individual sensors) or 'sensors' (for groups of sensors), and an optional 'title'. + - Example output: + [ + {"title": "Title 1", "sensor": }, + {"title": "Title 2", "sensors": [, ]}, + {"title": None, "sensors": [, ]}, + {"title": None, "sensor": }, + {"title": None, "sensor": } + ] + + If the 'sensors_to_show' attribute is missing, the function defaults to showing two of the asset's sensors, grouped together if they share the same unit, or separately if not. + + Unauthorized sensors are filtered out, and a warning is logged. Only sensors the user has permission to access are included in the final result. + """ + + if self.attributes is not None: + self.sensors_to_show = self.attributes.get("sensors_to_show", []) + else: + self.sensors_to_show = [] + + if not self.has_attribute("sensors_to_show"): + sensors_to_show = self.sensors[:2] + if ( + len(sensors_to_show) == 2 + and sensors_to_show[0].unit == sensors_to_show[1].unit + ): + # Sensors are shown together (e.g. they can share the same y-axis) + return [{"title": None, "sensors": sensors_to_show}] + # Otherwise, show separately + return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] + + sensor_ids_to_show = self.get_attribute("sensors_to_show") + # Import the schema for validation + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + + sensors_to_show_schema = SensorsToShowSchema() + + # Deserialize the sensor_ids_to_show using SensorsToShowSchema + standardized_sensors_to_show = sensors_to_show_schema.deserialize( + sensor_ids_to_show + ) + + sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) + + # Only allow showing sensors from assets owned by the user's organization, + # except in play mode, where any sensor may be shown + accounts = [self.owner] if self.owner is not None else None + if current_app.config.get("FLEXMEASURES_MODE") == "play": + from flexmeasures.data.models.user import Account + + accounts = db.session.scalars(select(Account)).all() + + from flexmeasures.data.services.sensors import get_sensors + + accessible_sensor_map = { + sensor.id: sensor + for sensor in get_sensors( + account=accounts, + include_public_assets=True, + sensor_id_allowlist=sensor_id_allowlist, + ) + } + + # Build list of sensor objects that are accessible + sensors_to_show = [] + missed_sensor_ids = [] + + for entry in standardized_sensors_to_show: + + title = entry.get("title") + sensors = entry.get("sensors") + + accessible_sensors = [ + accessible_sensor_map.get(sid) + for sid in sensors + if sid in accessible_sensor_map + ] + inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] + missed_sensor_ids.extend(inaccessible) + if accessible_sensors: + sensors_to_show.append({"title": title, "sensors": accessible_sensors}) + + if missed_sensor_ids: + current_app.logger.warning( + f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}." + ) + return sensors_to_show + @property def asset_type(self) -> GenericAssetType: """This property prepares for dropping the "generic" prefix later""" @@ -458,7 +579,7 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ - processed_sensors_to_show = validate_sesnors_to_show(self) + processed_sensors_to_show = self.validate_sesnors_to_show() sensors = flatten_unique(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -641,7 +762,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]: 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - return self.get_timerange(validate_sesnors_to_show(self)) + return self.get_timerange(self.validate_sesnors_to_show()) @classmethod def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa F821 diff --git a/flexmeasures/ui/crud/assets/views.py b/flexmeasures/ui/crud/assets/views.py index c4453d7d8..08ae87083 100644 --- a/flexmeasures/ui/crud/assets/views.py +++ b/flexmeasures/ui/crud/assets/views.py @@ -1,4 +1,5 @@ from __future__ import annotations +import json from flask import url_for, current_app, request from flask_classful import FlaskView, route from flask_security import login_required, current_user @@ -113,6 +114,14 @@ def get(self, id: str, **kwargs): get_asset_response = InternalApi().get(url_for("AssetAPI:fetch_one", id=id)) asset_dict = get_asset_response.json() + + # set sensors to show to list from string, this is not currently being used on the frontend from my knowledge + # it may be better popped of instead, but will leave it hear for now + if asset_dict.get("sensors_to_show") and not isinstance( + asset_dict.get("sensors_to_show"), list + ): + asset_dict["sensors_to_show"] = json.loads(asset_dict["sensors_to_show"]) + asset = process_internal_api_response(asset_dict, int(id), make_obj=True) asset_form = AssetForm() diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 7c520346a..f10a6195d 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -8,11 +8,7 @@ import importlib import pkgutil -from sqlalchemy import select from flask import current_app -from flask_security import current_user - -from flexmeasures.data import db def delete_key_recursive(value, key): @@ -181,130 +177,3 @@ def find_classes_modules(module, superclass, skiptest=True): def get_classes_module(module, superclass, skiptest=True) -> dict: return dict(find_classes_modules(module, superclass, skiptest=skiptest)) - - -def validate_sesnors_to_show( - asset, -) -> list[dict[str, str | None | "Sensor" | list["Sensor"]]]: # noqa: F821 - """ - Sensors to show, as defined by the sensors_to_show attribute. - - Sensors to show are defined as a list of sensor IDs, which are set by the "sensors_to_show" field in the asset's "attributes" column. - Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets. - In play mode, sensors from different accounts can be added. - - Sensor IDs can be nested to denote that sensors should be 'shown together', for example, layered rather than vertically concatenated. - Additionally, each row of sensors can be accompanied by a title. - If no title is provided, `"title": None` will be assigned in the returned dictionary. - - How to interpret 'shown together' is technically left up to the function returning chart specifications, as are any restrictions regarding which sensors can be shown together, such as: - - Whether they should share the same unit - - Whether they should share the same name - - Whether they should belong to different assets - - For example, this input denotes showing sensors 42 and 44 together: - - sensors_to_show = [40, 35, 41, [42, 44], 43, 45] - - And this input denotes showing sensors 42 and 44 together with a custom title: - - sensors_to_show = [ - {"title": "Title 1", "sensor": 40}, - {"title": "Title 2", "sensors": [41, 42]}, - [43, 44], 45, 46 - ] - - In both cases, the returned format will contain sensor objects mapped to their respective sensor IDs, as follows: - - [ - {"title": "Title 1", "sensor": }, - {"title": "Title 2", "sensors": [, ]}, - {"title": None, "sensors": [, ]}, - {"title": None, "sensor": }, - {"title": None, "sensor": } - ] - - In case the `sensors_to_show` field is missing, it defaults to two of the asset's sensors. These will be shown together (e.g., sharing the same y-axis) if they share the same unit; otherwise, they will be shown separately. - - Sensors are validated to ensure they are accessible by the user. If certain sensors are inaccessible, they will be excluded from the result, and a warning will be logged. The function only returns sensors that the user has permission to view. - """ - old_sensors_to_show = ( - asset.sensors_to_show - ) # Used to check if sensors_to_show was updated - if asset.attributes is not None: - asset.sensors_to_show = asset.attributes.get("sensors_to_show", []) - else: - asset.sensors_to_show = [] - - if not asset.sensors_to_show or asset.sensors_to_show == []: - sensors_to_show = asset.sensors[:2] - if ( - len(sensors_to_show) == 2 - and sensors_to_show[0].unit == sensors_to_show[1].unit - ): - # Sensors are shown together (e.g. they can share the same y-axis) - return [{"title": None, "sensors": sensors_to_show}] - # Otherwise, show separately - return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] - - sensor_ids_to_show = asset.sensors_to_show - # Import the schema for validation - from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema - - sensors_to_show_schema = SensorsToShowSchema() - - # Deserialize the sensor_ids_to_show using SensorsToShowSchema - standardized_sensors_to_show = sensors_to_show_schema.deserialize( - sensor_ids_to_show - ) - - sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) - - # Only allow showing sensors from assets owned by the user's organization, - # except in play mode, where any sensor may be shown - accounts = [asset.owner] if asset.owner is not None else None - if current_app.config.get("FLEXMEASURES_MODE") == "play": - from flexmeasures.data.models.user import Account - - accounts = db.session.scalars(select(Account)).all() - - from flexmeasures.data.services.sensors import get_sensors - - accessible_sensor_map = { - sensor.id: sensor - for sensor in get_sensors( - account=accounts, - include_public_assets=True, - sensor_id_allowlist=sensor_id_allowlist, - ) - } - - # Build list of sensor objects that are accessible - sensors_to_show = [] - missed_sensor_ids = [] - - for entry in standardized_sensors_to_show: - - title = entry.get("title") - sensors = entry.get("sensors") - - accessible_sensors = [ - accessible_sensor_map.get(sid) - for sid in sensors - if sid in accessible_sensor_map - ] - inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] - missed_sensor_ids.extend(inaccessible) - if accessible_sensors: - sensors_to_show.append({"title": title, "sensors": accessible_sensors}) - - if missed_sensor_ids: - current_app.logger.warning( - f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {asset.id}, as it is not accessible to user {current_user}." - ) - # check if sensors_to_show was updated - if old_sensors_to_show != asset.sensors_to_show: - asset.attributes["sensors_to_show"] = standardized_sensors_to_show - db.session.commit() - - return sensors_to_show From ab9c0f6680051b126049aa3eed76f689c05ddc23 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 3 Oct 2024 12:24:46 +0100 Subject: [PATCH 14/24] fix: wrong fuction name Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 0426d11de..46982b789 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -579,7 +579,7 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ - processed_sensors_to_show = self.validate_sesnors_to_show() + processed_sensors_to_show = self.validate_sensors_to_show() sensors = flatten_unique(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -762,7 +762,7 @@ def timerange_of_sensors_to_show(self) -> dict[str, datetime]: 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - return self.get_timerange(self.validate_sesnors_to_show()) + return self.get_timerange(self.validate_sensors_to_show()) @classmethod def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa F821 From 66bfecbf913873d50ba9b95aef1c1e1ba4219b6e Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 4 Oct 2024 20:38:14 +0100 Subject: [PATCH 15/24] refactor: relocate dict processing - Work in progress Signed-off-by: joshuaunity --- documentation/changelog.rst | 11 ++++++----- flexmeasures/data/models/generic_assets.py | 7 ++++--- flexmeasures/ui/crud/assets/utils.py | 3 +++ flexmeasures/ui/crud/assets/views.py | 9 +-------- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 8dc2da4a5..6769a9580 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,16 +7,20 @@ FlexMeasures Changelog v0.24.0 | October XX, 2024 ============================ +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + + New features ------------- -* The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 `_ and `PR #1193 `_ -* Speed up loading the users page, by making the pagination backend-based and adding support for that in the API [see `PR #1160 `] +* The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 `_ and `PR #1193 `_] +* Speed up loading the users page, by making the pagination backend-based and adding support for that in the API [see `PR #1160 `_] * X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 `_] Infrastructure / Support ---------------------- * For MacOS developers, install HiGHS solver automatically [see `PR #1187 `_] +* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data from parent source(attributes field) [see `PR #1200 `_] Bugfixes ----------- @@ -28,13 +32,10 @@ v0.23.0 | September 18, 2024 .. note:: Read more on these features on `the FlexMeasures blog `_. -.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). - New features ------------- * New chart type on sensor page: histogram [see `PR #1143 `_] * Add basic sensor info to sensor page [see `PR #1115 `_] -* Add dedicated ``sensors_to_show`` field to asset model and logic to migrate data from parent source(attributes field) [see `PR #1200 `_] * Add `Statistics` table on the sensor page and also add `api/v3_0/sensors//stats` endpoint to get sensor statistics [see `PR #1116 `_] * Support adding custom titles to the graphs on the asset page, by extending the ``sensors_to_show`` format [see `PR #1125 `_ and `PR #1177 `_] * Support zoom-in action on the asset and sensor charts [see `PR #1130 `_] diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 46982b789..85f50d5e0 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -179,7 +179,6 @@ def validate_sensors_to_show( This function ensures that the 'sensors_to_show' attribute: 1. Follows the latest format, even if the data in the database uses an older format. 2. Contains only sensors that the user has access to (based on the current asset, account, or public availability). - 3. Returns a list of dictionaries where each dictionary contains either a single sensor or a group of sensors with an optional title. Steps: - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly. @@ -223,7 +222,7 @@ def validate_sensors_to_show( else: self.sensors_to_show = [] - if not self.has_attribute("sensors_to_show"): + if not self.sensors_to_show: sensors_to_show = self.sensors[:2] if ( len(sensors_to_show) == 2 @@ -234,7 +233,7 @@ def validate_sensors_to_show( # Otherwise, show separately return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] - sensor_ids_to_show = self.get_attribute("sensors_to_show") + sensor_ids_to_show = self.sensors_to_show # Import the schema for validation from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema @@ -581,6 +580,7 @@ def chart( """ processed_sensors_to_show = self.validate_sensors_to_show() sensors = flatten_unique(processed_sensors_to_show) + for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -656,6 +656,7 @@ def search_beliefs( bdf_dict = {} if sensors is None: sensors = self.sensors + for sensor in sensors: bdf_dict[sensor] = sensor.search_beliefs( event_starts_after=event_starts_after, diff --git a/flexmeasures/ui/crud/assets/utils.py b/flexmeasures/ui/crud/assets/utils.py index 9056558ce..baeb727a2 100644 --- a/flexmeasures/ui/crud/assets/utils.py +++ b/flexmeasures/ui/crud/assets/utils.py @@ -112,6 +112,9 @@ def expunge_asset(): **{ **asset_data, **{"attributes": json.loads(asset_data.get("attributes", "{}"))}, + **{ + "sensors_to_show": json.loads(asset_data.get("sensors_to_show", [])) + }, } ) # TODO: use schema? if "generic_asset_type_id" in asset_data: diff --git a/flexmeasures/ui/crud/assets/views.py b/flexmeasures/ui/crud/assets/views.py index 08ae87083..c9e0850e0 100644 --- a/flexmeasures/ui/crud/assets/views.py +++ b/flexmeasures/ui/crud/assets/views.py @@ -1,5 +1,5 @@ from __future__ import annotations -import json + from flask import url_for, current_app, request from flask_classful import FlaskView, route from flask_security import login_required, current_user @@ -115,13 +115,6 @@ def get(self, id: str, **kwargs): get_asset_response = InternalApi().get(url_for("AssetAPI:fetch_one", id=id)) asset_dict = get_asset_response.json() - # set sensors to show to list from string, this is not currently being used on the frontend from my knowledge - # it may be better popped of instead, but will leave it hear for now - if asset_dict.get("sensors_to_show") and not isinstance( - asset_dict.get("sensors_to_show"), list - ): - asset_dict["sensors_to_show"] = json.loads(asset_dict["sensors_to_show"]) - asset = process_internal_api_response(asset_dict, int(id), make_obj=True) asset_form = AssetForm() From 544931057205c72defb5601b9d28ca9618416f7e Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 4 Oct 2024 20:49:52 +0100 Subject: [PATCH 16/24] fix: pass validated sensors_to_show instead of raw json data with IDs Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 54301d029..84c500c62 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -454,7 +454,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): Data for use in charts (in case you have the chart specs already). """ - sensors = flatten_unique(asset.sensors_to_show) + sensors = flatten_unique(asset.validate_sensors_to_show()) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//auditlog") From 7aee7b6e337fdae9b3135e8b9be9c69ba66f078b Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 7 Oct 2024 11:05:26 +0100 Subject: [PATCH 17/24] fix: proper fallback value for json loading Signed-off-by: joshuaunity --- flexmeasures/ui/crud/assets/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/ui/crud/assets/utils.py b/flexmeasures/ui/crud/assets/utils.py index baeb727a2..a1b2832e4 100644 --- a/flexmeasures/ui/crud/assets/utils.py +++ b/flexmeasures/ui/crud/assets/utils.py @@ -113,7 +113,9 @@ def expunge_asset(): **asset_data, **{"attributes": json.loads(asset_data.get("attributes", "{}"))}, **{ - "sensors_to_show": json.loads(asset_data.get("sensors_to_show", [])) + "sensors_to_show": json.loads( + asset_data.get("sensors_to_show", "[]") + ) }, } ) # TODO: use schema? @@ -179,6 +181,8 @@ def get_assets_by_account(account_id: int | str | None) -> list[GenericAsset]: ) else: get_assets_response = InternalApi().get(url_for("AssetAPI:public")) + + print("get_assets_by_account_func===================", get_assets_response.json()) return [ process_internal_api_response(ad, make_obj=True) for ad in get_assets_response.json() From d0487774862eff268e02fad24fe50adf35eb663f Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 7 Oct 2024 11:29:49 +0100 Subject: [PATCH 18/24] chore: removeing unused code Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 3 --- flexmeasures/ui/crud/assets/utils.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 85f50d5e0..6b6b90d7b 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -142,9 +142,6 @@ class GenericAsset(db.Model, AuthModelMixin): ), ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def __acl__(self): """ All logged-in users can read if the asset is public. diff --git a/flexmeasures/ui/crud/assets/utils.py b/flexmeasures/ui/crud/assets/utils.py index a1b2832e4..a412835df 100644 --- a/flexmeasures/ui/crud/assets/utils.py +++ b/flexmeasures/ui/crud/assets/utils.py @@ -181,8 +181,6 @@ def get_assets_by_account(account_id: int | str | None) -> list[GenericAsset]: ) else: get_assets_response = InternalApi().get(url_for("AssetAPI:public")) - - print("get_assets_by_account_func===================", get_assets_response.json()) return [ process_internal_api_response(ad, make_obj=True) for ad in get_assets_response.json() From 4af2598d5997f604f56c1e8ac0f37b1a22a57669 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 9 Oct 2024 10:38:51 +0100 Subject: [PATCH 19/24] refactor: read form sensors_to_show model field direct Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 6b6b90d7b..b21267f96 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -214,10 +214,11 @@ def validate_sensors_to_show( Unauthorized sensors are filtered out, and a warning is logged. Only sensors the user has permission to access are included in the final result. """ - if self.attributes is not None: - self.sensors_to_show = self.attributes.get("sensors_to_show", []) - else: - self.sensors_to_show = [] + self.sensors_to_show = ( + self.attributes.get("sensors_to_show", []) + if self.sensors_to_show is None + else self.sensors_to_show + ) if not self.sensors_to_show: sensors_to_show = self.sensors[:2] From 61b894244b188dbe0ee7b6334a63583478417ec1 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:49:51 +0100 Subject: [PATCH 20/24] Sensorstoshow input (#1208) * chore: in progress Signed-off-by: joshuaunity * chore: added to changelog Signed-off-by: joshuaunity --------- Signed-off-by: joshuaunity --- documentation/changelog.rst | 1 + flexmeasures/ui/crud/assets/forms.py | 1 + flexmeasures/ui/templates/crud/asset.html | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 95a30827a..ed21133bd 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,7 @@ New features * X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 `_] * Speed up loading the accounts page,by making the pagination backend-based and adding support for that in the API [see `PR #1196 `_] * Simplify and Globalize toasts in the flexmeasures project [see `PR #1207 _`] +* Added an input field to edit the raw JSON data of the sensors_to_show model field [see `PR #1208 _`] Infrastructure / Support ---------------------- diff --git a/flexmeasures/ui/crud/assets/forms.py b/flexmeasures/ui/crud/assets/forms.py index f5e6c3870..3381754ba 100644 --- a/flexmeasures/ui/crud/assets/forms.py +++ b/flexmeasures/ui/crud/assets/forms.py @@ -44,6 +44,7 @@ class AssetForm(FlaskForm): render_kw={"placeholder": "--Click the map or enter a longitude--"}, ) attributes = StringField("Other attributes (JSON)", default="{}") + sensors_to_show = StringField("Sensors to show (JSON)", default="[]") production_price_sensor_id = SelectField( "Production price sensor", coerce=int, validate_choice=False ) diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index 318879d74..bde823478 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -184,6 +184,15 @@

Edit {{ asset.name }}

{% endfor %} +
+ {{ asset_form.sensors_to_show.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.sensors_to_show(class_="form-control") }} + {% for error in asset_form.errors.sensors_to_show %} + [{{error}}] + {% endfor %} +
+
(Click map to edit latitude and longitude in form)
From 5580ebcd5a379fccd14f4fd55b5b11ffb288b759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 9 Oct 2024 22:31:15 +0200 Subject: [PATCH 21/24] remove changelog entry for adding form field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index ed21133bd..95a30827a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,6 @@ New features * X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 `_] * Speed up loading the accounts page,by making the pagination backend-based and adding support for that in the API [see `PR #1196 `_] * Simplify and Globalize toasts in the flexmeasures project [see `PR #1207 _`] -* Added an input field to edit the raw JSON data of the sensors_to_show model field [see `PR #1208 _`] Infrastructure / Support ---------------------- From 308284e1548cc021aac906d94d24f2adfd9368fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 9 Oct 2024 22:55:40 +0200 Subject: [PATCH 22/24] do not rely on attributes anymore for this; add comment on defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/models/generic_assets.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index b21267f96..daa68a9b5 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -213,13 +213,7 @@ def validate_sensors_to_show( Unauthorized sensors are filtered out, and a warning is logged. Only sensors the user has permission to access are included in the final result. """ - - self.sensors_to_show = ( - self.attributes.get("sensors_to_show", []) - if self.sensors_to_show is None - else self.sensors_to_show - ) - + # If not set, use defaults (show first 2 sensors) if not self.sensors_to_show: sensors_to_show = self.sensors[:2] if ( From ccfdd9e8a1fb15eedff6a23c822af67c7bf8a235 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 10 Oct 2024 09:47:09 +0100 Subject: [PATCH 23/24] feat: repopulate the attrributes when sensors_to_show field is removed from db downgrade Signed-off-by: joshuaunity --- ...dd_sensors_to_show_field_in_asset_model.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py index fccf85484..9ea0be2d5 100644 --- a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py +++ b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py @@ -57,5 +57,45 @@ def upgrade(): def downgrade(): + # Initialize the connection + conn = op.get_bind() + + # Define the generic_asset table + generic_asset_table = sa.Table( + "generic_asset", + sa.MetaData(), + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("attributes", sa.JSON), + sa.Column("sensors_to_show", sa.JSON), + ) + + # Select the rows where we want to update the attributes field + select_stmt = select( + generic_asset_table.c.id, + generic_asset_table.c.sensors_to_show, + generic_asset_table.c.attributes, + ) + results = conn.execute(select_stmt) + + # Iterate over the results and update the attributes column + for row in results: + asset_id, sensors_to_show_data, attributes_data = row + + # Ensure attributes_data is a dictionary, default to empty dict if None + if attributes_data is None: + attributes_data = {} + + # Add sensors_to_show back into the attributes field + attributes_data["sensors_to_show"] = sensors_to_show_data + + # Update the attributes field with the modified data + update_stmt = ( + generic_asset_table.update() + .where(generic_asset_table.c.id == asset_id) + .values(attributes=attributes_data) + ) + conn.execute(update_stmt) + + # After migrating data back to the attributes field, drop the sensors_to_show column with op.batch_alter_table("generic_asset", schema=None) as batch_op: batch_op.drop_column("sensors_to_show") From a0909d9b1f981d4570e92498cdac4a0b65d88b16 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 10 Oct 2024 09:58:35 +0100 Subject: [PATCH 24/24] chore: clear uneeded comments Signed-off-by: joshuaunity --- ...50e23e3aa54_add_sensors_to_show_field_in_asset_model.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py index 9ea0be2d5..3839c0158 100644 --- a/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py +++ b/flexmeasures/data/migrations/versions/950e23e3aa54_add_sensors_to_show_field_in_asset_model.py @@ -57,10 +57,8 @@ def upgrade(): def downgrade(): - # Initialize the connection conn = op.get_bind() - # Define the generic_asset table generic_asset_table = sa.Table( "generic_asset", sa.MetaData(), @@ -69,7 +67,6 @@ def downgrade(): sa.Column("sensors_to_show", sa.JSON), ) - # Select the rows where we want to update the attributes field select_stmt = select( generic_asset_table.c.id, generic_asset_table.c.sensors_to_show, @@ -77,18 +74,14 @@ def downgrade(): ) results = conn.execute(select_stmt) - # Iterate over the results and update the attributes column for row in results: asset_id, sensors_to_show_data, attributes_data = row - # Ensure attributes_data is a dictionary, default to empty dict if None if attributes_data is None: attributes_data = {} - # Add sensors_to_show back into the attributes field attributes_data["sensors_to_show"] = sensors_to_show_data - # Update the attributes field with the modified data update_stmt = ( generic_asset_table.update() .where(generic_asset_table.c.id == asset_id)