Skip to content

Commit

Permalink
Fix/sensor stats timezone (#1213)
Browse files Browse the repository at this point in the history 
* refactor: simple sorting *is* possible

Signed-off-by: F.N. Claessen <felix@seita.nl>

* feat: improve names of stats (no over-capitalization)

Signed-off-by: F.N. Claessen <felix@seita.nl>

* feat: report last event end rather than last event start

Signed-off-by: F.N. Claessen <felix@seita.nl>

* feat: report event timing stats with UTC offset and conforming to ISO8601 rather than with a timezone that isn't that of the sensor

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: update test

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: 'status' not part of response anymore (this came from flask_json's `as_json`)

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Revert "fix: 'status' not part of response anymore (this came from flask_json's `as_json`)"

This reverts commit 71673ea.

* Revert "refactor: simple sorting *is* possible"

This reverts commit 08fa56f

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: no offset shorthand in test

Signed-off-by: F.N. Claessen <felix@seita.nl>

* refactor: simple sorting **is** still possible (I hope)

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: end is sensor resolution of 10 minutes later

Signed-off-by: F.N. Claessen <felix@seita.nl>

* style: black

Signed-off-by: F.N. Claessen <felix@seita.nl>

* refactor: move config setting to Config class

Signed-off-by: F.N. Claessen <felix@seita.nl>

* remove: redundant setting (the Config superclass has the same default)

Signed-off-by: F.N. Claessen <felix@seita.nl>

* docs: changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>

---------

Signed-off-by: F.N. Claessen <felix@seita.nl>
Signed-off-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com>
  • Loading branch information
@Flix6x
Flix6x authored Oct 17, 2024
1 parent 8427259 commit 008801b
Show file tree
Hide file tree
Showing 5 changed files with 43 additions and 49 deletions.
1 change: 1 addition & 0 deletions documentation/changelog.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Bugfixes
-----------
* The UI footer now stays at the bottom even on pages with little content [see `PR #1204 <https://github.com/FlexMeasures/flexmeasures/pull/1204>`_]
* Correct stroke dash (based on source type) for forecasts made by forecasters included in FlexMeasures [see `PR #1211 <https://www.github.com/FlexMeasures/flexmeasures/pull/1211>`_]
* Show the correct UTC offset for the data's time span as shown under sensor stats in the UI [see `PR #1213 <https://github.com/FlexMeasures/flexmeasures/pull/1213>`_]


v0.23.0 | September 18, 2024
Expand Down
16 changes: 8 additions & 8 deletions flexmeasures/api/v3_0/tests/test_sensors_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
def test_fetch_sensor_stats(
client, setup_api_test_data: dict[str, Sensor], requesting_user, db
):
# gas sensor is setup in add_gas_measurements
# gas sensor is set up in add_gas_measurements
sensor_id = 1
with QueryCounter(db.session.connection()) as counter1:
response = client.get(
Expand All @@ -363,10 +363,10 @@ def test_fetch_sensor_stats(
"Test Supplier User",
]
for source, record in response_content.items():
assert record["min_event_start"] == "Sat, 01 May 2021 22:00:00 GMT"
assert record["max_event_start"] == "Sat, 01 May 2021 22:20:00 GMT"
assert record["min_value"] == 91.3
assert record["max_value"] == 92.1
assert record["First event start"] == "2021-05-01T22:00:00+00:00"
assert record["Last event end"] == "2021-05-01T22:30:00+00:00"
assert record["Min value"] == 91.3
assert record["Max value"] == 92.1
if source == "Test Supplier User":
# values are: 91.3, 91.7, 92.1
sum_values = 275.1
Expand All @@ -377,12 +377,12 @@ def test_fetch_sensor_stats(
count_values = 3
mean_value = 91.7
assert math.isclose(
record["mean_value"], mean_value, rel_tol=1e-5
record["Mean value"], mean_value, rel_tol=1e-5
), f"mean_value is close to {mean_value}"
assert math.isclose(
record["sum_values"], sum_values, rel_tol=1e-5
record["Sum over values"], sum_values, rel_tol=1e-5
), f"sum_values is close to {sum_values}"
assert record["count_values"] == count_values
assert record["Number of values"] == count_values

with QueryCounter(db.session.connection()) as counter2:
response = client.get(
Expand Down
23 changes: 16 additions & 7 deletions flexmeasures/data/services/sensors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from isodate import duration_isoformat
import time
from timely_beliefs import BeliefsDataFrame
import pandas as pd

from humanize.time import naturaldelta

Expand Down Expand Up @@ -394,14 +395,22 @@ def _get_sensor_stats(sensor: Sensor, ttl_hash=None) -> dict:
sum_values,
count_values,
) = row
first_event_start = (
pd.Timestamp(min_event_start).tz_convert(sensor.timezone).isoformat()
)
last_event_end = (
pd.Timestamp(max_event_start + sensor.event_resolution)
.tz_convert(sensor.timezone)
.isoformat()
)
stats[data_source] = {
"min_event_start": min_event_start,
"max_event_start": max_event_start,
"min_value": min_value,
"max_value": max_value,
"mean_value": mean_value,
"sum_values": sum_values,
"count_values": count_values,
"First event start": first_event_start,
"Last event end": last_event_end,
"Min value": min_value,
"Max value": max_value,
"Mean value": mean_value,
"Sum over values": sum_values,
"Number of values": count_values,
}
return stats

Expand Down
50 changes: 17 additions & 33 deletions flexmeasures/ui/templates/views/sensors.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,15 @@ <h5>Properties</h5>
<td>{{ sensor.unit }}</td>
</tr>
<tr>
<th>Event Resolution</th>
<th>Event resolution</th>
<td>{{ sensor.event_resolution }}</td>
</tr>
<tr>
<th>Timezone</th>
<td>{{ sensor.timezone }}</td>
</tr>
<tr>
<th>Knowledge Horizon Type</th>
<th>Knowledge horizon type</th>
<td>{{ sensor.knowledge_horizon_fnc }}</td>
</tr>
</table>
Expand Down Expand Up @@ -178,38 +178,22 @@ <h5 id="statsHeader">Statistics</h5>
function updateTable(stats) {
tableBody.innerHTML = ''; // Clear the table body

// This list is used to format the table content. Simple sorting is not possible.
const keys = [
"min_event_start",
"max_event_start",
"min_value",
"max_value",
"mean_value",
"sum_values",
"count_values"
];

keys.forEach(key => {
if (key in stats) {
// Create a display key by removing underscores and capitalizing words
const displayKey = key.replace(/_/g, ' ').replace(/\b\w/g, char => char.toUpperCase());

const row = document.createElement('tr');
const keyCell = document.createElement('th');
const valueCell = document.createElement('td');

keyCell.textContent = displayKey;
// Round value to 2 decimal points if it's a number
if (typeof stats[key] === 'number' & key != 'count_values') {
valueCell.textContent = stats[key].toFixed(4);
} else {
valueCell.textContent = stats[key];
}

row.appendChild(keyCell);
row.appendChild(valueCell);
tableBody.appendChild(row);
Object.entries(stats).forEach(([key, val]) => {
const row = document.createElement('tr');
const keyCell = document.createElement('th');
const valueCell = document.createElement('td');

keyCell.textContent = key;
// Round value to 2 decimal points if it's a number
if (typeof val === 'number' & key != 'Number of values') {
valueCell.textContent = val.toFixed(4);
} else {
valueCell.textContent = val;
}

row.appendChild(keyCell);
row.appendChild(valueCell);
tableBody.appendChild(row);
});
}
});
Expand Down
2 changes: 1 addition & 1 deletion flexmeasures/utils/config_defaults.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ class Config(object):
# todo: expand with other js versions used in FlexMeasures
)
FLEXMEASURES_JSON_COMPACT = False
JSON_SORT_KEYS = False

FLEXMEASURES_FALLBACK_REDIRECT: bool = False

Expand Down Expand Up @@ -191,7 +192,6 @@ class DevelopmentConfig(Config):
# PRESERVE_CONTEXT_ON_EXCEPTION: bool = False # might need this to make our transaction handling work in debug mode
FLEXMEASURES_MODE: str = "development"
FLEXMEASURES_PROFILE_REQUESTS: bool = True
FLEXMEASURES_JSON_COMPACT = False


class TestingConfig(Config):
Expand Down

0 comments on commit 008801b

Please sign in to comment.