diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2818e6528..6fef53f86 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -15,6 +15,7 @@ New features * The data chart on the asset page splits up its color-coded sensor legend when showing more than 7 sensors, becoming a legend per subplot [see `PR #1176 `_ and `PR #1193 `_] * Speed up loading the users page, by making the pagination backend-based and adding support for that in the API [see `PR #1160 `_] * X-axis labels in CLI plots show datetime values in a readable and informative format [see `PR #1172 `_] +* Enhanced API for listing sensors: Added filtering and pagination on sensor index endpoint and created new endpoint to get all sensors under an asset [see `PR #1191 `_ ] * Speed up loading the accounts page,by making the pagination backend-based and adding support for that in the API [see `PR #1196 `_] * Speed up loading the account detail page by by switching to server-side pagination for assets, replacing client-side pagination [see `PR #1202 `_] * Simplify and Globalize toasts in the flexmeasures project [see `PR #1207 _`] diff --git a/flexmeasures/api/common/schemas/generic_assets.py b/flexmeasures/api/common/schemas/generic_assets.py index ccf41e348..cdd5fe81b 100644 --- a/flexmeasures/api/common/schemas/generic_assets.py +++ b/flexmeasures/api/common/schemas/generic_assets.py @@ -1,9 +1,7 @@ from __future__ import annotations -from shlex import join, split - from flask import abort -from marshmallow import fields, ValidationError +from marshmallow import fields from sqlalchemy import select from flexmeasures.data import db @@ -25,17 +23,3 @@ def _deserialize(self, asset_id: int, attr, obj, **kwargs) -> GenericAsset: def _serialize(self, asset: GenericAsset, attr, data, **kwargs) -> int: return asset.id - - -class SearchFilterField(fields.Str): - """Field that represents a search filter.""" - - def _deserialize(self, value, attr, data, **kwargs) -> list[str]: - try: - search_terms = split(value) - except ValueError as e: - raise ValidationError(str(e)) - return search_terms - - def _serialize(self, value: list[str], attr, obj, **kwargs) -> str: - return join(value) diff --git a/flexmeasures/api/common/schemas/search.py b/flexmeasures/api/common/schemas/search.py new file mode 100644 index 000000000..fa24b0e01 --- /dev/null +++ b/flexmeasures/api/common/schemas/search.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from shlex import join, split + +from marshmallow import fields, ValidationError + + +class SearchFilterField(fields.Str): + """Field that represents a search filter.""" + + def _deserialize(self, value, attr, data, **kwargs) -> list[str]: + try: + search_terms = split(value) + except ValueError as e: + raise ValidationError(str(e)) + return search_terms + + def _serialize(self, value: list[str], attr, obj, **kwargs) -> str: + return join(value) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index dccc23c4d..30c1e75d2 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -1,5 +1,5 @@ from flask import abort -from marshmallow import fields +from marshmallow import fields, ValidationError from sqlalchemy import select from flexmeasures.data import db @@ -9,6 +9,7 @@ EntityAddressException, ) from flexmeasures.data.models.time_series import Sensor +from flexmeasures.utils.unit_utils import is_valid_unit class EntityAddressValidationError(FMValidationError): @@ -80,3 +81,15 @@ def _serialize(self, value: Sensor, attr, data, **kwargs): return value.entity_address_fm0 else: return value.entity_address + + +class UnitField(fields.Str): + """Field that represents a unit.""" + + def _deserialize(self, value, attr, data, **kwargs) -> str: + if not is_valid_unit(value): + raise ValidationError(f"Invalid unit: {value}") + return value + + def _serialize(self, value: str, attr, obj, **kwargs) -> str: + return value diff --git a/flexmeasures/api/v3_0/accounts.py b/flexmeasures/api/v3_0/accounts.py index ccb069c5b..06406c888 100644 --- a/flexmeasures/api/v3_0/accounts.py +++ b/flexmeasures/api/v3_0/accounts.py @@ -20,7 +20,7 @@ from flexmeasures.data.services.accounts import get_accounts, get_audit_log_records from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.data.schemas.account import AccountSchema -from flexmeasures.api.common.schemas.generic_assets import SearchFilterField +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.utils.time_utils import server_now """ diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 97375dd6f..9ecf593d5 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1,5 +1,6 @@ from __future__ import annotations import json +from humanize import naturaldelta from flask import current_app from flask_classful import FlaskView, route @@ -22,19 +23,21 @@ from flexmeasures.data.queries.generic_assets import query_assets_by_search_terms from flexmeasures.data.schemas import AwareDateTimeField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema -from flexmeasures.api.common.schemas.generic_assets import ( - AssetIdField, - SearchFilterField, -) +from flexmeasures.api.common.schemas.generic_assets import AssetIdField +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.ui.utils.view_utils import set_session_variables from flexmeasures.auth.policy import check_access from werkzeug.exceptions import Forbidden, Unauthorized +from flexmeasures.data.schemas.sensors import SensorSchema +from flexmeasures.data.models.time_series import Sensor asset_schema = AssetSchema() assets_schema = AssetSchema(many=True) +sensor_schema = SensorSchema() +sensors_schema = SensorSchema(many=True) partial_asset_schema = AssetSchema(partial=True, exclude=["account_id"]) @@ -155,10 +158,6 @@ def index( if all_accessible or include_public: filter_statement = filter_statement | GenericAsset.account_id.is_(None) - num_records = db.session.scalar( - select(func.count(GenericAsset.id)).where(filter_statement) - ) - query = query_assets_by_search_terms( search_terms=filter, filter_statement=filter_statement ) @@ -171,6 +170,9 @@ def index( select_pagination: SelectPagination = db.paginate( query, per_page=per_page, page=page ) + num_records = db.session.scalar( + select(func.count(GenericAsset.id)).filter(filter_statement) + ) response = { "data": asset_schema.dump(select_pagination.items, many=True), "num-records": num_records, @@ -179,6 +181,110 @@ def index( return response, 200 + @route( + "//sensors", + methods=["GET"], + ) + @use_kwargs( + { + "asset": AssetIdField(data_key="id"), + }, + location="path", + ) + @use_kwargs( + { + "page": fields.Int( + required=False, validate=validate.Range(min=1), dump_default=1 + ), + "per_page": fields.Int( + required=False, validate=validate.Range(min=1), dump_default=10 + ), + }, + location="query", + ) + @as_json + def asset_sensors( + self, + id: int, + asset: GenericAsset | None, + page: int | None = None, + per_page: int | None = None, + ): + """ + List all sensors under an asset. + + .. :quickref: Asset; Return all sensors under an asset. + + This endpoint returns all sensors under an asset. + + The endpoint supports pagination of the asset list using the `page` and `per_page` query parameters. + + - If the `page` parameter is not provided, all sensors are returned, without pagination information. The result will be a list of sensors. + - If a `page` parameter is provided, the response will be paginated, showing a specific number of assets per page as defined by `per_page` (default is 10). + The response schema for pagination is inspired by https://datatables.net/manual/server-side#Returned-data + + + **Example response** + + An example of one asset being returned in a paginated response: + + .. sourcecode:: json + + { + "data" : [ + { + "id": 1, + "name": "Test battery", + "latitude": 10, + "longitude": 100, + "account_id": 2, + "generic_asset_type": {"id": 1, "name": "battery"} + } + ], + "num-records" : 1, + "filtered-records" : 1 + + } + + If no pagination is requested, the response only consists of the list under the "data" key. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + query_statement = Sensor.generic_asset_id == asset.id + + query = select(Sensor).filter(query_statement) + + select_pagination: SelectPagination = db.paginate( + query, per_page=per_page, page=page + ) + + num_records = db.session.scalar( + select(func.count(Sensor.id)).where(query_statement) + ) + + sensors_response: list = [ + { + **sensor_schema.dump(sensor), + "event_resolution": naturaldelta(sensor.event_resolution), + } + for sensor in select_pagination.items + ] + + response = { + "data": sensors_response, + "num-records": num_records, + "filtered-records": select_pagination.total, + } + + return response, 200 + @route("/public", methods=["GET"]) @as_json def public(self): diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 252e3a6a5..75d027e5c 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,17 +1,18 @@ from __future__ import annotations +import isodate from datetime import datetime, timedelta from flask import current_app, url_for from flask_classful import FlaskView, route from flask_json import as_json from flask_security import auth_required -import isodate from marshmallow import fields, ValidationError +import marshmallow.validate as validate from rq.job import Job, NoSuchJobError from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args, use_kwargs -from sqlalchemy import delete +from sqlalchemy import delete, select, or_ from flexmeasures.api.common.responses import ( request_processed, @@ -29,6 +30,7 @@ ) from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures.auth.policy import check_access from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data import db from flexmeasures.data.models.audit_log import AssetAuditLog @@ -37,8 +39,10 @@ from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField +from flexmeasures.api.common.schemas.search import SearchFilterField +from flexmeasures.api.common.schemas.sensors import UnitField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField -from flexmeasures.data.services.sensors import get_sensors, get_sensor_stats +from flexmeasures.data.services.sensors import get_sensor_stats from flexmeasures.data.services.scheduling import ( create_scheduling_job, get_data_source_for_job, @@ -65,18 +69,37 @@ class SensorAPI(FlaskView): "account": AccountIdField( data_key="account_id", load_default=AccountIdField.load_current ), + "all_accessible": fields.Boolean(required=False, missing=False), + "page": fields.Int( + required=False, validate=validate.Range(min=1), default=1 + ), + "per_page": fields.Int( + required=False, validate=validate.Range(min=1), default=10 + ), + "filter": SearchFilterField(required=False, default=None), + "unit": UnitField(required=False, default=None), }, location="query", ) @permission_required_for_context("read", ctx_arg_name="account") @as_json - def index(self, account: Account): + def index( + self, + account: Account, + all_accessible: bool = False, + page: int | None = None, + per_page: int | None = None, + filter: list[str] | None = None, + unit: str | None = None, + ): """API endpoint to list all sensors of an account. .. :quickref: Sensor; Download sensor list This endpoint returns all accessible sensors. Accessible sensors are sensors in the same account as the current user. + Alternatively, you can use the `all_accessible` query parameter to list sensors from all assets that the `current_user` has read access to, as well as all public assets. The default value is `false`. + Only admins can use this endpoint to fetch sensors from a different account (by using the `account_id` query parameter). **Example response** @@ -85,7 +108,8 @@ def index(self, account: Account): .. sourcecode:: json - [ + { + "data" : [ { "entity_address": "ea1.2021-01.io.flexmeasures.company:fm1.42", "event_resolution": PT15M, @@ -95,7 +119,12 @@ def index(self, account: Account): "unit": "m\u00b3/h" "id": 2 } - ] + ], + "num-records" : 1, + "filtered-records" : 1 + } + + If no pagination is requested, the response only consists of the list under the "data" key. :reqheader Authorization: The authentication token :reqheader Content-Type: application/json @@ -106,8 +135,69 @@ def index(self, account: Account): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - sensors = get_sensors(account=account) - return sensors_schema.dump(sensors), 200 + if isinstance(account, list): + accounts = account + else: + accounts: list = [account] if account else [] + account_ids: list = [acc.id for acc in accounts] + + filter_statement = GenericAsset.account_id.in_(account_ids) + + if all_accessible is not None: + consultancy_account_ids: list = [ + acc.consultancy_account_id for acc in accounts + ] + account_ids.extend(consultancy_account_ids) + filter_statement = or_( + filter_statement, + GenericAsset.account_id.is_(None), + ) + + sensor_query = ( + select(Sensor) + .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) + .join(Account, GenericAsset.owner) + .filter(filter_statement) + ) + + if filter is not None: + sensor_query = sensor_query.filter( + or_( + *( + or_( + Sensor.name.ilike(f"%{term}%"), + Account.name.ilike(f"%{term}%"), + ) + for term in filter + ) + ) + ) + + if unit: + sensor_query = sensor_query.filter(Sensor.unit == unit) + + sensors = ( + db.session.scalars(sensor_query).all() + if page is None + else db.paginate(sensor_query, per_page=per_page, page=page).items + ) + + sensors = [sensor for sensor in sensors if check_access(sensor, "read") is None] + + sensors_response = sensors_schema.dump(sensors) + + # Return appropriate response for paginated or non-paginated data + if page is None: + return sensors_response, 200 + else: + num_records = len(db.session.execute(sensor_query).scalars().all()) + select_pagination = db.paginate(sensor_query, per_page=per_page, page=page) + response = { + "data": sensors_response, + "num-records": num_records, + "filtered-records": select_pagination.total, + } + return response, 200 @route("/data", methods=["POST"]) @use_args( diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 5e43f0155..d44716036 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -9,6 +9,7 @@ from flexmeasures.data.services.users import find_user_by_email from flexmeasures.api.tests.utils import get_auth_token, UserContext, AccountContext from flexmeasures.api.v3_0.tests.utils import get_asset_post_data +from flexmeasures.utils.unit_utils import is_valid_unit @pytest.mark.parametrize( @@ -118,6 +119,28 @@ def test_get_assets( assert turbine["account_id"] == setup_accounts["Supplier"].id +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_fetch_asset_sensors(client, setup_api_test_data, requesting_user): + """ + Retrieve all sensors associated with a specific asset. + + This test checks for these metadata fields and the number of sensors returned, as well as + confirming that the response is a list of dictionaries, each containing a valid unit. + """ + asset_id = setup_api_test_data["some gas sensor"].generic_asset_id + response = client.get(url_for("AssetAPI:asset_sensors", id=asset_id)) + + print("Server responded with:\n%s" % response.json) + + assert response.status_code == 200 + assert response.json["status"] == 200 + assert isinstance(response.json["data"], list) + assert isinstance(response.json["data"][0], dict) + assert is_valid_unit(response.json["data"][0]["unit"]) + assert response.json["num-records"] == 3 + assert response.json["filtered-records"] == 3 + + @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) def test_get_asset_with_children(client, add_asset_with_children, requesting_user): """ diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 10a5a8e91..99b6ec6d3 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -13,10 +13,107 @@ from flexmeasures.data.schemas.sensors import SensorSchema from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.tests.utils import QueryCounter +from flexmeasures.utils.unit_utils import is_valid_unit + sensor_schema = SensorSchema() +@pytest.mark.parametrize( + "requesting_user, search_by, search_value, exp_sensor_name, exp_num_results, all_accessible, use_pagination", + [ + ( + "test_supplier_user_4@seita.nl", + "unit", + "°C", + "some temperature sensor", + 2, + True, + False, + ), + ( + "test_supplier_user_4@seita.nl", + "unit", + "m³/h", + "some gas sensor", + 1, + True, + False, + ), + ( + "test_supplier_user_4@seita.nl", + None, + None, + "some temperature sensor", + 3, + True, + True, + ), + ( + "test_supplier_user_4@seita.nl", + "filter", + "'some temperature sensor'", + "some temperature sensor", + 1, + False, + False, + ), + ], + indirect=["requesting_user"], +) +def test_fetch_sensors( + client, + setup_api_test_data, + requesting_user, + search_by, + search_value, + exp_sensor_name, + exp_num_results, + all_accessible, + use_pagination, +): + """ + Retrieve all sensors. + + Our user here is admin, so is allowed to see all sensors. + Pagination is tested only in passing, we should test filtering and page > 1 + """ + query = {search_by: search_value} + + if use_pagination: + query["page"] = 1 + + if search_by == "unit": + query["unit"] = search_value + elif search_by == "filter": + query["filter"] = search_value + + if all_accessible: + query["all_accessible"] = True + + response = client.get( + url_for("SensorAPI:index"), + query_string=query, + ) + + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + + if use_pagination: + assert isinstance(response.json["data"][0], dict) + assert is_valid_unit(response.json["data"][0]["unit"]) + assert response.json["num-records"] == exp_num_results + assert response.json["filtered-records"] == exp_num_results + else: + assert isinstance(response.json, list) + assert is_valid_unit(response.json[0]["unit"]) + assert response.json[0]["name"] == exp_sensor_name + assert len(response.json) == exp_num_results + + if search_by == "unit": + assert response.json[0]["unit"] == search_value + + @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index a54afafd8..8015f45ff 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -15,7 +15,7 @@ from flexmeasures.data.models.audit_log import AuditLog from flexmeasures.data.models.user import User as UserModel, Account from flexmeasures.api.common.schemas.users import AccountIdField, UserIdField -from flexmeasures.api.common.schemas.generic_assets import SearchFilterField +from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.api.v3_0.assets import get_accessible_accounts from flexmeasures.data.queries.users import query_users_by_search_terms from flexmeasures.data.schemas.account import AccountSchema diff --git a/flexmeasures/data/queries/sensors.py b/flexmeasures/data/queries/sensors.py index 3336f65a6..ea0194c17 100644 --- a/flexmeasures/data/queries/sensors.py +++ b/flexmeasures/data/queries/sensors.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlalchemy.sql import Select, select +from sqlalchemy import select, Select from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.queries.utils import potentially_limit_assets_query_to_account diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 47de8f4e2..acf8f10c4 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -38,12 +38,12 @@ def get_sensors( :param sensor_name_allowlist: optionally, allow only sensors whose name is in this list """ sensor_query = sa.select(Sensor) - if account is None: - account_ids = [] - elif isinstance(account, list): - account_ids = [account.id for account in account] + if isinstance(account, list): + accounts = account else: - account_ids = [account.id] + accounts: list = [account] if account else [] + account_ids: list = [acc.id for acc in accounts] + sensor_query = sensor_query.join( GenericAsset, GenericAsset.id == Sensor.generic_asset_id ).filter(Sensor.generic_asset_id == GenericAsset.id) @@ -60,6 +60,7 @@ def get_sensors( sensor_query = sensor_query.filter(Sensor.id.in_(sensor_id_allowlist)) if sensor_name_allowlist: sensor_query = sensor_query.filter(Sensor.name.in_(sensor_name_allowlist)) + return db.session.scalars(sensor_query).all() diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index bde823478..a623e54c4 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -283,42 +283,10 @@

Edit {{ asset.name }}

All sensors for {{ asset.name }}

+
- - - - - - - - - - - - - {% for sensor in asset.sensors: %} - - - - - - - - - {% endfor %} - + +
NameIDUnitResolutionEntity addressURL
- {{ sensor.name }} - - {{ sensor.id }} - - {{ sensor.unit }} - - {{ sensor.event_resolution | naturalized_timedelta }} - - {{ sensor.entity_address }} - - /sensors/{{ sensor.id }} -
@@ -434,4 +402,91 @@

All child assets for {{ asset.name }}

}); + + +{% block paginate_tables_script %} {{ super() }} {% endblock %} {% endblock %}