From 008801b145d957ff1d3954dec96dafe0d477d00d Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:05:53 +0200 Subject: [PATCH] Fix/sensor stats timezone (#1213) * refactor: simple sorting *is* possible Signed-off-by: F.N. Claessen * feat: improve names of stats (no over-capitalization) Signed-off-by: F.N. Claessen * feat: report last event end rather than last event start Signed-off-by: F.N. Claessen * feat: report event timing stats with UTC offset and conforming to ISO8601 rather than with a timezone that isn't that of the sensor Signed-off-by: F.N. Claessen * fix: update test Signed-off-by: F.N. Claessen * fix: 'status' not part of response anymore (this came from flask_json's `as_json`) Signed-off-by: F.N. Claessen * Revert "fix: 'status' not part of response anymore (this came from flask_json's `as_json`)" This reverts commit 71673ea2559f0f31232090ea47c48eb85526168b. * Revert "refactor: simple sorting *is* possible" This reverts commit 08fa56fc Signed-off-by: F.N. Claessen * fix: no offset shorthand in test Signed-off-by: F.N. Claessen * refactor: simple sorting **is** still possible (I hope) Signed-off-by: F.N. Claessen * fix: end is sensor resolution of 10 minutes later Signed-off-by: F.N. Claessen * style: black Signed-off-by: F.N. Claessen * refactor: move config setting to Config class Signed-off-by: F.N. Claessen * remove: redundant setting (the Config superclass has the same default) Signed-off-by: F.N. Claessen * docs: changelog entry Signed-off-by: F.N. Claessen --------- Signed-off-by: F.N. Claessen Signed-off-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> --- documentation/changelog.rst | 1 + .../api/v3_0/tests/test_sensors_api.py | 16 +++--- flexmeasures/data/services/sensors.py | 23 ++++++--- flexmeasures/ui/templates/views/sensors.html | 50 +++++++------------ flexmeasures/utils/config_defaults.py | 2 +- 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9d146b6cf..afdd2ac53 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -29,6 +29,7 @@ Bugfixes ----------- * The UI footer now stays at the bottom even on pages with little content [see `PR #1204 `_] * Correct stroke dash (based on source type) for forecasts made by forecasters included in FlexMeasures [see `PR #1211 `_] +* Show the correct UTC offset for the data's time span as shown under sensor stats in the UI [see `PR #1213 `_] v0.23.0 | September 18, 2024 diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 99b6ec6d3..7ca9da878 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -347,7 +347,7 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db): def test_fetch_sensor_stats( client, setup_api_test_data: dict[str, Sensor], requesting_user, db ): - # gas sensor is setup in add_gas_measurements + # gas sensor is set up in add_gas_measurements sensor_id = 1 with QueryCounter(db.session.connection()) as counter1: response = client.get( @@ -363,10 +363,10 @@ def test_fetch_sensor_stats( "Test Supplier User", ] for source, record in response_content.items(): - assert record["min_event_start"] == "Sat, 01 May 2021 22:00:00 GMT" - assert record["max_event_start"] == "Sat, 01 May 2021 22:20:00 GMT" - assert record["min_value"] == 91.3 - assert record["max_value"] == 92.1 + assert record["First event start"] == "2021-05-01T22:00:00+00:00" + assert record["Last event end"] == "2021-05-01T22:30:00+00:00" + assert record["Min value"] == 91.3 + assert record["Max value"] == 92.1 if source == "Test Supplier User": # values are: 91.3, 91.7, 92.1 sum_values = 275.1 @@ -377,12 +377,12 @@ def test_fetch_sensor_stats( count_values = 3 mean_value = 91.7 assert math.isclose( - record["mean_value"], mean_value, rel_tol=1e-5 + record["Mean value"], mean_value, rel_tol=1e-5 ), f"mean_value is close to {mean_value}" assert math.isclose( - record["sum_values"], sum_values, rel_tol=1e-5 + record["Sum over values"], sum_values, rel_tol=1e-5 ), f"sum_values is close to {sum_values}" - assert record["count_values"] == count_values + assert record["Number of values"] == count_values with QueryCounter(db.session.connection()) as counter2: response = client.get( diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index acf8f10c4..a3172c1dc 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -8,6 +8,7 @@ from isodate import duration_isoformat import time from timely_beliefs import BeliefsDataFrame +import pandas as pd from humanize.time import naturaldelta @@ -394,14 +395,22 @@ def _get_sensor_stats(sensor: Sensor, ttl_hash=None) -> dict: sum_values, count_values, ) = row + first_event_start = ( + pd.Timestamp(min_event_start).tz_convert(sensor.timezone).isoformat() + ) + last_event_end = ( + pd.Timestamp(max_event_start + sensor.event_resolution) + .tz_convert(sensor.timezone) + .isoformat() + ) stats[data_source] = { - "min_event_start": min_event_start, - "max_event_start": max_event_start, - "min_value": min_value, - "max_value": max_value, - "mean_value": mean_value, - "sum_values": sum_values, - "count_values": count_values, + "First event start": first_event_start, + "Last event end": last_event_end, + "Min value": min_value, + "Max value": max_value, + "Mean value": mean_value, + "Sum over values": sum_values, + "Number of values": count_values, } return stats diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 5872a5e6d..9b58e0252 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -72,7 +72,7 @@
Properties
{{ sensor.unit }} - Event Resolution + Event resolution {{ sensor.event_resolution }} @@ -80,7 +80,7 @@
Properties
{{ sensor.timezone }} - Knowledge Horizon Type + Knowledge horizon type {{ sensor.knowledge_horizon_fnc }} @@ -178,38 +178,22 @@
Statistics
function updateTable(stats) { tableBody.innerHTML = ''; // Clear the table body - // This list is used to format the table content. Simple sorting is not possible. - const keys = [ - "min_event_start", - "max_event_start", - "min_value", - "max_value", - "mean_value", - "sum_values", - "count_values" - ]; - - keys.forEach(key => { - if (key in stats) { - // Create a display key by removing underscores and capitalizing words - const displayKey = key.replace(/_/g, ' ').replace(/\b\w/g, char => char.toUpperCase()); - - const row = document.createElement('tr'); - const keyCell = document.createElement('th'); - const valueCell = document.createElement('td'); - - keyCell.textContent = displayKey; - // Round value to 2 decimal points if it's a number - if (typeof stats[key] === 'number' & key != 'count_values') { - valueCell.textContent = stats[key].toFixed(4); - } else { - valueCell.textContent = stats[key]; - } - - row.appendChild(keyCell); - row.appendChild(valueCell); - tableBody.appendChild(row); + Object.entries(stats).forEach(([key, val]) => { + const row = document.createElement('tr'); + const keyCell = document.createElement('th'); + const valueCell = document.createElement('td'); + + keyCell.textContent = key; + // Round value to 2 decimal points if it's a number + if (typeof val === 'number' & key != 'Number of values') { + valueCell.textContent = val.toFixed(4); + } else { + valueCell.textContent = val; } + + row.appendChild(keyCell); + row.appendChild(valueCell); + tableBody.appendChild(row); }); } }); diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 1e92d0ba5..b28556b9b 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -133,6 +133,7 @@ class Config(object): # todo: expand with other js versions used in FlexMeasures ) FLEXMEASURES_JSON_COMPACT = False + JSON_SORT_KEYS = False FLEXMEASURES_FALLBACK_REDIRECT: bool = False @@ -191,7 +192,6 @@ class DevelopmentConfig(Config): # PRESERVE_CONTEXT_ON_EXCEPTION: bool = False # might need this to make our transaction handling work in debug mode FLEXMEASURES_MODE: str = "development" FLEXMEASURES_PROFILE_REQUESTS: bool = True - FLEXMEASURES_JSON_COMPACT = False class TestingConfig(Config):