diff --git a/django_project/gap/admin/preferences.py b/django_project/gap/admin/preferences.py index c70cbf97..147dc869 100644 --- a/django_project/gap/admin/preferences.py +++ b/django_project/gap/admin/preferences.py @@ -52,8 +52,9 @@ class PreferencesAdmin(admin.ModelAdmin): } ), ( - 'Dask Config', { + 'Ingestor Config', { 'fields': ( + 'ingestor_config', 'dask_threads_num', ) } diff --git a/django_project/gap/fixtures/8.dataset_attribute.json b/django_project/gap/fixtures/8.dataset_attribute.json index 0491e9ab..5d37b083 100755 --- a/django_project/gap/fixtures/8.dataset_attribute.json +++ b/django_project/gap/fixtures/8.dataset_attribute.json @@ -1043,5 +1043,27 @@ "source_unit": 8, "ensembles": false } + }, + { + "model": "gap.datasetattribute", + "pk": 96, + "fields": { + "dataset": 6, + "attribute": 6, + "source": "solarGHI", + "source_unit": 7, + "ensembles": false + } + }, + { + "model": "gap.datasetattribute", + "pk": 97, + "fields": { + "dataset": 4, + "attribute": 6, + "source": "solar_radiation", + "source_unit": 7, + "ensembles": false + } } ] \ No newline at end of file diff --git a/django_project/gap/ingestor/base.py b/django_project/gap/ingestor/base.py index 8323d424..2c415e57 100644 --- a/django_project/gap/ingestor/base.py +++ b/django_project/gap/ingestor/base.py @@ -72,7 +72,17 @@ def get_config(self, name: str, default_value = None): class BaseZarrIngestor(BaseIngestor): - """Base Ingestor class for Zarr product.""" + """Base Ingestor class for Zarr product. + + Available config for ingestor: + datasourcefile_id: Id of existing DataSourceFile + datasourcefile_zarr_exists: Indicates that Zarr exists on s3, + or create new one + datasourcefile_name: set the zarr name if creating new zarr + remove_temp_file: remove temporary from the collector + use_latest_datasource: Default to True, + always use the latest DataSourceFile + """ default_zarr_name = f'{uuid.uuid4()}.zarr' diff --git a/django_project/gap/ingestor/tio_shortterm.py b/django_project/gap/ingestor/tio_shortterm.py index 3f9abeaa..8e5c4844 100644 --- a/django_project/gap/ingestor/tio_shortterm.py +++ b/django_project/gap/ingestor/tio_shortterm.py @@ -182,7 +182,8 @@ class TioShortTermIngestor(BaseZarrIngestor): 'precipitation_probability', 'humidity_maximum', 'humidity_minimum', - 'wind_speed_avg' + 'wind_speed_avg', + 'solar_radiation' ] def __init__(self, session: IngestorSession, working_dir: str = '/tmp'): diff --git a/django_project/gap/migrations/0030_preferences_ingestor_config.py b/django_project/gap/migrations/0030_preferences_ingestor_config.py new file mode 100644 index 00000000..eef6b7eb --- /dev/null +++ b/django_project/gap/migrations/0030_preferences_ingestor_config.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-10-11 11:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gap', '0029_datasourcefilecache_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='preferences', + name='ingestor_config', + field=models.JSONField(blank=True, default=dict, help_text='Dict of ProviderName and AdditionalConfig; AdditionalConfig will be passed to the Ingestor Session.', null=True), + ), + ] diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py index fc9a10bf..86358307 100644 --- a/django_project/gap/models/preferences.py +++ b/django_project/gap/models/preferences.py @@ -109,6 +109,17 @@ class Preferences(SingletonModel): ) ) + # ingestor config + ingestor_config = models.JSONField( + default=dict, + blank=True, + null=True, + help_text=( + 'Dict of ProviderName and AdditionalConfig; ' + 'AdditionalConfig will be passed to the Ingestor Session.' + ) + ) + class Meta: # noqa: D106 verbose_name_plural = "preferences" diff --git a/django_project/gap/tasks/collector.py b/django_project/gap/tasks/collector.py index bc85a8de..493eabf4 100644 --- a/django_project/gap/tasks/collector.py +++ b/django_project/gap/tasks/collector.py @@ -8,12 +8,12 @@ from celery.utils.log import get_task_logger from core.celery import app -from gap.models.dataset import ( +from gap.models import ( + Preferences, + Provider, Dataset, DatasetStore, - DataSourceFile -) -from gap.models.ingestor import ( + DataSourceFile, IngestorType, IngestorSession, CollectorSession @@ -48,6 +48,18 @@ def run_cbam_collector_session(): ) +def _get_ingestor_config_from_preferences(provider: Provider) -> dict: + """Retrieve additional config for a provider. + + :param provider: provider + :type provider: Provider + :return: additional config for Ingestor + :rtype: dict + """ + config = Preferences.load().ingestor_config + return config.get(provider.name, {}) + + def _do_run_zarr_collector( dataset: Dataset, collector_session: CollectorSession, ingestor_type): @@ -67,18 +79,25 @@ def _do_run_zarr_collector( collector_session.refresh_from_db() total_file = collector_session.dataset_files.count() if total_file > 0: - # find latest DataSourceFile - data_source = DataSourceFile.objects.filter( - dataset=dataset, - format=DatasetStore.ZARR, - is_latest=True - ).last() additional_conf = {} - if data_source: - additional_conf = { - 'datasourcefile_id': data_source.id, - 'datasourcefile_zarr_exists': True - } + config = _get_ingestor_config_from_preferences(dataset.provider) + + use_latest_datasource = config.get('use_latest_datasource', True) + if use_latest_datasource: + # find latest DataSourceFile + data_source = DataSourceFile.objects.filter( + dataset=dataset, + format=DatasetStore.ZARR, + is_latest=True + ).last() + if data_source: + additional_conf = { + 'datasourcefile_id': data_source.id, + 'datasourcefile_zarr_exists': True + } + additional_conf.update(config) + + # create session and trigger the task session = IngestorSession.objects.create( ingestor_type=ingestor_type, trigger_task=False, diff --git a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py index e0b6e712..39aef6c5 100644 --- a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py +++ b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py @@ -136,6 +136,7 @@ def test_collector_one_grid(self, mock_timezone): self.assertEqual(len(zip_file.filelist), 1) _file = zip_file.open(f'grid-{grid.id}.json') _data = json.loads(_file.read().decode('utf-8')) + print(_data) self.assertEqual( _data, { @@ -157,7 +158,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 0, 'humidity_maximum': 84, 'humidity_minimum': 69, - 'wind_speed_avg': 4.77 + 'wind_speed_avg': 4.77, + 'solar_radiation': None } }, { @@ -170,7 +172,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 80, 'humidity_minimum': 71, - 'wind_speed_avg': 4.35 + 'wind_speed_avg': 4.35, + 'solar_radiation': None } }, { @@ -183,7 +186,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 79, 'humidity_minimum': 73, - 'wind_speed_avg': 5.58 + 'wind_speed_avg': 5.58, + 'solar_radiation': None } }, { @@ -196,7 +200,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 78, 'humidity_minimum': 72, - 'wind_speed_avg': 5.74 + 'wind_speed_avg': 5.74, + 'solar_radiation': None } }, { @@ -209,7 +214,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 76, 'humidity_minimum': 70, - 'wind_speed_avg': 5.09 + 'wind_speed_avg': 5.09, + 'solar_radiation': None } }, { @@ -222,7 +228,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 76, 'humidity_minimum': 72, - 'wind_speed_avg': 4.01 + 'wind_speed_avg': 4.01, + 'solar_radiation': None } }, { @@ -235,7 +242,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 0, 'humidity_maximum': 76, 'humidity_minimum': 70, - 'wind_speed_avg': 3.82 + 'wind_speed_avg': 3.82, + 'solar_radiation': None } }, { @@ -248,7 +256,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 78, 'humidity_minimum': 72, - 'wind_speed_avg': 4.12 + 'wind_speed_avg': 4.12, + 'solar_radiation': None } }, { @@ -261,7 +270,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 80, 'humidity_minimum': 74, - 'wind_speed_avg': 5.29 + 'wind_speed_avg': 5.29, + 'solar_radiation': None } }, { @@ -274,7 +284,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 80, 'humidity_minimum': 73, - 'wind_speed_avg': 4.96 + 'wind_speed_avg': 4.96, + 'solar_radiation': None } }, { @@ -287,7 +298,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 77, 'humidity_minimum': 68, - 'wind_speed_avg': 4.1 + 'wind_speed_avg': 4.1, + 'solar_radiation': None } }, { @@ -300,7 +312,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 78, 'humidity_minimum': 70, - 'wind_speed_avg': 4.42 + 'wind_speed_avg': 4.42, + 'solar_radiation': None } }, { @@ -313,7 +326,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 78, 'humidity_minimum': 72, - 'wind_speed_avg': 4.52 + 'wind_speed_avg': 4.52, + 'solar_radiation': None } }, { @@ -326,7 +340,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 78, 'humidity_minimum': 72, - 'wind_speed_avg': 4.74 + 'wind_speed_avg': 4.74, + 'solar_radiation': None } }, { @@ -339,7 +354,8 @@ def test_collector_one_grid(self, mock_timezone): 'precipitation_probability': 5, 'humidity_maximum': 77.83, 'humidity_minimum': 72.77, - 'wind_speed_avg': 3.17 + 'wind_speed_avg': 3.17, + 'solar_radiation': None } } ] diff --git a/docs/src/developer/api/guide/measurment.md b/docs/src/developer/api/guide/measurment.md index aebfa832..f88f6076 100644 --- a/docs/src/developer/api/guide/measurment.md +++ b/docs/src/developer/api/guide/measurment.md @@ -86,6 +86,7 @@ TomorrowNow provides access to the data through a RESTful API, available at http | Short-term Forecast | Humidity Maximum | | % | humidity_maximum | | Short-term Forecast | Humidity Minimum | | % | humidity_minimum | | Short-term Forecast | Wind Speed Average | | m/s | wind_speed_avg | +| Short-term Forecast | Solar radiation | | Wh/m2 | solar_radiation | | **Historical Reanalysis** | | Historical Reanalysis | Min Total Temperature | Minimum temperature (0000:2300) | °C | min_temperature | | Historical Reanalysis | Min Day Temperature | Minimum day-time temperature (0600:1800) | °C | min_day_temperature |