From 4dc4e04d3f30fce4afc18930fe12be63a028fb36 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 20 Sep 2024 17:43:36 +0100 Subject: [PATCH 01/24] feat: search filter on sensor API Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/sensors.py | 18 +++++++++-- flexmeasures/data/services/sensors.py | 43 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 252e3a6a5..4cc7a295b 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -38,7 +38,11 @@ from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField -from flexmeasures.data.services.sensors import get_sensors, get_sensor_stats +from flexmeasures.data.services.sensors import ( + get_sensors, + get_sensor_stats, + get_all_accessible_sensors, +) from flexmeasures.data.services.scheduling import ( create_scheduling_job, get_data_source_for_job, @@ -65,12 +69,16 @@ class SensorAPI(FlaskView): "account": AccountIdField( data_key="account_id", load_default=AccountIdField.load_current ), + "all_accessible": fields.Boolean(required=False, missing=False), + "filter": fields.Str(load_default=None), }, location="query", ) @permission_required_for_context("read", ctx_arg_name="account") @as_json - def index(self, account: Account): + def index( + self, account: Account, all_accessible: bool = False, filter: str | None = None + ): """API endpoint to list all sensors of an account. .. :quickref: Sensor; Download sensor list @@ -106,7 +114,11 @@ def index(self, account: Account): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - sensors = get_sensors(account=account) + if all_accessible: + sensors = get_all_accessible_sensors(account, filter) + else: + sensors = get_sensors(account, filter=filter) + return sensors_schema.dump(sensors), 200 @route("/data", methods=["POST"]) diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 173a57f95..a841f75e2 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -22,6 +22,39 @@ from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.reporting import StatusSchema from flexmeasures.utils.time_utils import server_now +from flexmeasures.auth.policy import check_access + + +def get_all_accessible_sensors(account: Account, filter: str | None = None): + accounts: list = [account] if account else [] + account_ids: list = [acc.id for acc in accounts] + consultancy_account_ids: list = [acc.consultancy_account_id for acc in accounts] + account_ids.extend(consultancy_account_ids) + + sensor_query = ( + sa.select(Sensor) + .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) + .join(Account, GenericAsset.owner) + .filter( + sa.or_( + GenericAsset.account_id.in_(account_ids), + GenericAsset.account_id.is_(None), + ) + ) + ) + + if filter: + filter_statement = f"%{filter}%" + sensor_query = sensor_query.filter( + sa.or_( + Sensor.name.ilike(filter_statement), + Account.name.ilike(filter_statement), + ) + ) + + sensors = db.session.scalars(sensor_query).all() + + return [sensor for sensor in sensors if check_access(sensor, "read") is None] def get_sensors( @@ -29,6 +62,7 @@ def get_sensors( include_public_assets: bool = False, sensor_id_allowlist: list[int] | None = None, sensor_name_allowlist: list[str] | None = None, + filter: str | None = None, ) -> list[Sensor]: """Return a list of Sensor objects that belong to the given account, and/or public sensors. @@ -60,6 +94,15 @@ def get_sensors( sensor_query = sensor_query.filter(Sensor.id.in_(sensor_id_allowlist)) if sensor_name_allowlist: sensor_query = sensor_query.filter(Sensor.name.in_(sensor_name_allowlist)) + + if filter: + print("===================filter:", filter) + sensor_query = sensor_query.join(Account, GenericAsset.owner).filter( + sa.or_( + Sensor.name.ilike(filter), + Account.name.ilike(filter), + ) + ) return db.session.scalars(sensor_query).all() From 92c58a0511fb6476ff727ddb718acce94c0e846b Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 24 Sep 2024 15:51:26 +0100 Subject: [PATCH 02/24] feat: sensor API pagintion and search Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/sensors.py | 102 +++++++++++++--- flexmeasures/data/queries/sensors.py | 23 +++- flexmeasures/data/services/sensors.py | 51 +------- flexmeasures/ui/templates/crud/asset.html | 134 ++++++++++++++++------ 4 files changed, 213 insertions(+), 97 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 4cc7a295b..c38b83a09 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,17 +1,20 @@ from __future__ import annotations +import isodate from datetime import datetime, timedelta +from humanize import naturaldelta from flask import current_app, url_for from flask_classful import FlaskView, route from flask_json import as_json from flask_security import auth_required -import isodate from marshmallow import fields, ValidationError +import marshmallow.validate as validate from rq.job import Job, NoSuchJobError from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args, use_kwargs -from sqlalchemy import delete +from sqlalchemy import delete, select, or_ +from flask_sqlalchemy.pagination import SelectPagination from flexmeasures.api.common.responses import ( request_processed, @@ -29,6 +32,7 @@ ) from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures.auth.policy import check_access from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data import db from flexmeasures.data.models.audit_log import AssetAuditLog @@ -37,12 +41,10 @@ from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField +from flexmeasures.api.common.schemas.generic_assets import SearchFilterField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField -from flexmeasures.data.services.sensors import ( - get_sensors, - get_sensor_stats, - get_all_accessible_sensors, -) +from flexmeasures.data.queries.sensors import query_sensors_by_search_terms +from flexmeasures.data.services.sensors import get_sensor_stats from flexmeasures.data.services.scheduling import ( create_scheduling_job, get_data_source_for_job, @@ -70,14 +72,25 @@ class SensorAPI(FlaskView): data_key="account_id", load_default=AccountIdField.load_current ), "all_accessible": fields.Boolean(required=False, missing=False), - "filter": fields.Str(load_default=None), + "page": fields.Int( + required=False, validate=validate.Range(min=1), default=1 + ), + "per_page": fields.Int( + required=False, validate=validate.Range(min=1), default=10 + ), + "filter": SearchFilterField(required=False, default=None), }, location="query", ) @permission_required_for_context("read", ctx_arg_name="account") @as_json def index( - self, account: Account, all_accessible: bool = False, filter: str | None = None + self, + account: Account, + all_accessible: bool = False, + page: int | None = None, + per_page: int | None = None, + filter: list[str] | None = None, ): """API endpoint to list all sensors of an account. @@ -114,12 +127,75 @@ def index( :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - if all_accessible: - sensors = get_all_accessible_sensors(account, filter) + accounts: list = [account] if account else [] + account_ids: list = [acc.id for acc in accounts] + + if all_accessible is not None: + consultancy_account_ids: list = [ + acc.consultancy_account_id for acc in accounts + ] + account_ids.extend(consultancy_account_ids) + + filter_statement = GenericAsset.account_id.in_(account_ids) + + sensor_query = ( + select(Sensor) + .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) + .join(Account, GenericAsset.owner) + .filter( + or_( + filter_statement, + GenericAsset.account_id.is_(None), + ) + ) + ) + + if filter is not None: + sensor_query = query_sensors_by_search_terms( + sensor_query, search_terms=filter + ) + + if page is None: + sensors = db.session.scalars(sensor_query).all() + if all_accessible: + sensors = [ + sensor for sensor in sensors if check_access(sensor, "read") is None + ] + + sensors_response: list = [ + { + **sensor_schema.dump(sensor), + "event_resolution": naturaldelta(sensor.event_resolution), + } + for sensor in sensors + ] + + return sensors_response, 200 else: - sensors = get_sensors(account, filter=filter) + select_pagination: SelectPagination = db.paginate( + sensor_query, per_page=per_page, page=page + ) + sensors = select_pagination.items + sensors = ( + [sensor for sensor in sensors if check_access(sensor, "read") is None] + if all_accessible + else sensors + ) + + sensors_response: list = [ + { + **sensor_schema.dump(sensor), + "event_resolution": naturaldelta(sensor.event_resolution), + } + for sensor in sensors + ] + response = { + "data": sensors_response, + "num-records": select_pagination.total, + "filtered-records": select_pagination.total, + } - return sensors_schema.dump(sensors), 200 + return response, 200 @route("/data", methods=["POST"]) @use_args( diff --git a/flexmeasures/data/queries/sensors.py b/flexmeasures/data/queries/sensors.py index 3336f65a6..5a633d6cb 100644 --- a/flexmeasures/data/queries/sensors.py +++ b/flexmeasures/data/queries/sensors.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlalchemy.sql import Select, select +from sqlalchemy import select, Select, or_, and_ from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.queries.utils import potentially_limit_assets_query_to_account @@ -65,3 +65,24 @@ def query_sensors_by_proximity( closest_sensor_query, account_id ) return closest_sensor_query + + +def query_sensors_by_search_terms( + query: Select, + search_terms: list[str] | None, +) -> Select: + if search_terms is not None: + from flexmeasures.data.models.user import Account + from flexmeasures.data.models.time_series import Sensor + + filter_statement = and_( + *( + or_( + Sensor.name.ilike(f"%{term}%"), + Account.name.ilike(f"%{term}%"), + ) + for term in search_terms + ) + ) + + return query.filter(filter_statement) diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index a841f75e2..3bd7a5cef 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -22,39 +22,6 @@ from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.reporting import StatusSchema from flexmeasures.utils.time_utils import server_now -from flexmeasures.auth.policy import check_access - - -def get_all_accessible_sensors(account: Account, filter: str | None = None): - accounts: list = [account] if account else [] - account_ids: list = [acc.id for acc in accounts] - consultancy_account_ids: list = [acc.consultancy_account_id for acc in accounts] - account_ids.extend(consultancy_account_ids) - - sensor_query = ( - sa.select(Sensor) - .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) - .join(Account, GenericAsset.owner) - .filter( - sa.or_( - GenericAsset.account_id.in_(account_ids), - GenericAsset.account_id.is_(None), - ) - ) - ) - - if filter: - filter_statement = f"%{filter}%" - sensor_query = sensor_query.filter( - sa.or_( - Sensor.name.ilike(filter_statement), - Account.name.ilike(filter_statement), - ) - ) - - sensors = db.session.scalars(sensor_query).all() - - return [sensor for sensor in sensors if check_access(sensor, "read") is None] def get_sensors( @@ -62,7 +29,6 @@ def get_sensors( include_public_assets: bool = False, sensor_id_allowlist: list[int] | None = None, sensor_name_allowlist: list[str] | None = None, - filter: str | None = None, ) -> list[Sensor]: """Return a list of Sensor objects that belong to the given account, and/or public sensors. @@ -72,12 +38,9 @@ def get_sensors( :param sensor_name_allowlist: optionally, allow only sensors whose name is in this list """ sensor_query = sa.select(Sensor) - if account is None: - account_ids = [] - elif isinstance(account, list): - account_ids = [account.id for account in account] - else: - account_ids = [account.id] + accounts: list = [account] if account else [] + account_ids: list = [acc.id for acc in accounts] + sensor_query = sensor_query.join( GenericAsset, GenericAsset.id == Sensor.generic_asset_id ).filter(Sensor.generic_asset_id == GenericAsset.id) @@ -95,14 +58,6 @@ def get_sensors( if sensor_name_allowlist: sensor_query = sensor_query.filter(Sensor.name.in_(sensor_name_allowlist)) - if filter: - print("===================filter:", filter) - sensor_query = sensor_query.join(Account, GenericAsset.owner).filter( - sa.or_( - Sensor.name.ilike(filter), - Account.name.ilike(filter), - ) - ) return db.session.scalars(sensor_query).all() diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index 318879d74..801b22a5a 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -274,42 +274,19 @@

Edit {{ asset.name }}

All sensors for {{ asset.name }}

+
+ +
- - - - - - - - - - - - - {% for sensor in asset.sensors: %} - - - - - - - - - {% endfor %} - + +
@@ -425,4 +402,91 @@

All child assets for {{ asset.name }}

}); + + +{% block paginate_tables_script %} {{ super() }} {% endblock %} {% endblock %} From 658b525b878d752db4305d8c7b0d8ceb312ee6fa Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 24 Sep 2024 18:32:11 +0100 Subject: [PATCH 03/24] refactor: reduce code repitition and code size Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/sensors.py | 53 +++++++++++++------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index c38b83a09..bc77221b4 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -14,7 +14,6 @@ from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args, use_kwargs from sqlalchemy import delete, select, or_ -from flask_sqlalchemy.pagination import SelectPagination from flexmeasures.api.common.responses import ( request_processed, @@ -155,46 +154,36 @@ def index( sensor_query, search_terms=filter ) - if page is None: - sensors = db.session.scalars(sensor_query).all() - if all_accessible: - sensors = [ - sensor for sensor in sensors if check_access(sensor, "read") is None - ] + sensors = ( + db.session.scalars(sensor_query).all() + if page is None + else db.paginate(sensor_query, per_page=per_page, page=page).items + ) - sensors_response: list = [ - { - **sensor_schema.dump(sensor), - "event_resolution": naturaldelta(sensor.event_resolution), - } - for sensor in sensors - ] + sensors = ( + [sensor for sensor in sensors if check_access(sensor, "read") is None] + if all_accessible + else sensors + ) + + sensors_response = [ + { + **sensor_schema.dump(sensor), + "event_resolution": naturaldelta(sensor.event_resolution), + } + for sensor in sensors + ] + # Return appropriate response for paginated or non-paginated data + if page is None: return sensors_response, 200 else: - select_pagination: SelectPagination = db.paginate( - sensor_query, per_page=per_page, page=page - ) - sensors = select_pagination.items - sensors = ( - [sensor for sensor in sensors if check_access(sensor, "read") is None] - if all_accessible - else sensors - ) - - sensors_response: list = [ - { - **sensor_schema.dump(sensor), - "event_resolution": naturaldelta(sensor.event_resolution), - } - for sensor in sensors - ] + select_pagination = db.paginate(sensor_query, per_page=per_page, page=page) response = { "data": sensors_response, "num-records": select_pagination.total, "filtered-records": select_pagination.total, } - return response, 200 @route("/data", methods=["POST"]) From 2114eca92f2bfe4c8d09371a8a934070df095a32 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 25 Sep 2024 11:48:29 +0100 Subject: [PATCH 04/24] refactor: request changes - relocated SearchField to dedicated file - worked on event resolution formating Signed-off-by: joshuaunity --- .../api/common/schemas/generic_assets.py | 18 +------- flexmeasures/api/common/schemas/search.py | 19 +++++++++ flexmeasures/api/v3_0/assets.py | 6 +-- flexmeasures/api/v3_0/sensors.py | 42 +++++++++++-------- flexmeasures/api/v3_0/users.py | 2 +- flexmeasures/ui/templates/crud/asset.html | 24 ++++++++++- 6 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 flexmeasures/api/common/schemas/search.py diff --git a/flexmeasures/api/common/schemas/generic_assets.py b/flexmeasures/api/common/schemas/generic_assets.py index ccf41e348..cdd5fe81b 100644 --- a/flexmeasures/api/common/schemas/generic_assets.py +++ b/flexmeasures/api/common/schemas/generic_assets.py @@ -1,9 +1,7 @@ from __future__ import annotations -from shlex import join, split - from flask import abort -from marshmallow import fields, ValidationError +from marshmallow import fields from sqlalchemy import select from flexmeasures.data import db @@ -25,17 +23,3 @@ def _deserialize(self, asset_id: int, attr, obj, **kwargs) -> GenericAsset: def _serialize(self, asset: GenericAsset, attr, data, **kwargs) -> int: return asset.id - - -class SearchFilterField(fields.Str): - """Field that represents a search filter.""" - - def _deserialize(self, value, attr, data, **kwargs) -> list[str]: - try: - search_terms = split(value) - except ValueError as e: - raise ValidationError(str(e)) - return search_terms - - def _serialize(self, value: list[str], attr, obj, **kwargs) -> str: - return join(value) diff --git a/flexmeasures/api/common/schemas/search.py b/flexmeasures/api/common/schemas/search.py new file mode 100644 index 000000000..fa24b0e01 --- /dev/null +++ b/flexmeasures/api/common/schemas/search.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from shlex import join, split + +from marshmallow import fields, ValidationError + + +class SearchFilterField(fields.Str): + """Field that represents a search filter.""" + + def _deserialize(self, value, attr, data, **kwargs) -> list[str]: + try: + search_terms = split(value) + except ValueError as e: + raise ValidationError(str(e)) + return search_terms + + def _serialize(self, value: list[str], attr, obj, **kwargs) -> str: + return join(value) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 54301d029..aa43dd989 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -23,10 +23,8 @@ from flexmeasures.data.queries.generic_assets import query_assets_by_search_terms from flexmeasures.data.schemas import AwareDateTimeField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema -from flexmeasures.api.common.schemas.generic_assets import ( - AssetIdField, - SearchFilterField, -) +from flexmeasures.api.common.schemas.search import SearchFilterField +from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.ui.utils.view_utils import set_session_variables diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index bc77221b4..fb314f7f8 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -2,7 +2,6 @@ import isodate from datetime import datetime, timedelta -from humanize import naturaldelta from flask import current_app, url_for from flask_classful import FlaskView, route @@ -40,7 +39,7 @@ from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField -from flexmeasures.api.common.schemas.generic_assets import SearchFilterField +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.queries.sensors import query_sensors_by_search_terms from flexmeasures.data.services.sensors import get_sensor_stats @@ -97,6 +96,8 @@ def index( This endpoint returns all accessible sensors. Accessible sensors are sensors in the same account as the current user. + Alternatively, you can use the `all_accessible` query parameter to list sensors from all assets that the `current_user` has read access to, as well as all public assets. The default value is `false`. + Only admins can use this endpoint to fetch sensors from a different account (by using the `account_id` query parameter). **Example response** @@ -105,7 +106,8 @@ def index( .. sourcecode:: json - [ + { + "data" : [ { "entity_address": "ea1.2021-01.io.flexmeasures.company:fm1.42", "event_resolution": PT15M, @@ -115,7 +117,12 @@ def index( "unit": "m\u00b3/h" "id": 2 } - ] + ], + "num-records" : 1, + "filtered-records" : 1 + } + + If no pagination is requested, the response only consists of the list under the "data" key. :reqheader Authorization: The authentication token :reqheader Content-Type: application/json @@ -126,7 +133,10 @@ def index( :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - accounts: list = [account] if account else [] + if isinstance(account, list): + accounts = account + else: + accounts: list = [account] if account else [] account_ids: list = [acc.id for acc in accounts] if all_accessible is not None: @@ -160,19 +170,17 @@ def index( else db.paginate(sensor_query, per_page=per_page, page=page).items ) - sensors = ( - [sensor for sensor in sensors if check_access(sensor, "read") is None] - if all_accessible - else sensors - ) + sensors = [sensor for sensor in sensors if check_access(sensor, "read") is None] - sensors_response = [ - { - **sensor_schema.dump(sensor), - "event_resolution": naturaldelta(sensor.event_resolution), - } - for sensor in sensors - ] + # sensors_response = [ + # { + # **sensor_schema.dump(sensor), + # "event_resolution": naturaldelta(sensor.event_resolution), + # } + # for sensor in sensors + # ] + + sensors_response = sensors_schema.dump(sensors) # Return appropriate response for paginated or non-paginated data if page is None: diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index ff6ea6926..e6c774d65 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -15,7 +15,7 @@ from flexmeasures.data.models.audit_log import AuditLog from flexmeasures.data.models.user import User as UserModel, Account from flexmeasures.api.common.schemas.users import AccountIdField, UserIdField -from flexmeasures.api.common.schemas.generic_assets import SearchFilterField +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.api.v3_0.assets import get_accessible_accounts from flexmeasures.data.queries.users import query_users_by_search_terms from flexmeasures.data.schemas.account import AccountSchema diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index 801b22a5a..8224af65d 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -402,7 +402,11 @@

All child assets for {{ asset.name }}

}); + -