From 4d484f79773f4f53ea1b4f69fb92668272dc2162 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Tue, 18 Jun 2024 17:57:08 +0700 Subject: [PATCH] Split ggmn and other data to download data --- .../0086_alter_downloadrequest_data_type.py | 18 +++++++ models/download_request.py | 4 +- tasks/data_file_cache/base_cache.py | 47 ++++++++++++------- tasks/data_file_cache/country_recache.py | 5 +- 4 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 migrations/0086_alter_downloadrequest_data_type.py diff --git a/migrations/0086_alter_downloadrequest_data_type.py b/migrations/0086_alter_downloadrequest_data_type.py new file mode 100644 index 0000000..49d1525 --- /dev/null +++ b/migrations/0086_alter_downloadrequest_data_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2024-06-18 10:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gwml2', '0085_view_well'), + ] + + operations = [ + migrations.AlterField( + model_name='downloadrequest', + name='data_type', + field=models.CharField(choices=[('GGMN', 'GGMN'), ('Well and Monitoring Data', 'Other data')], default='GGMN', max_length=512, verbose_name='Data type'), + ), + ] diff --git a/models/download_request.py b/models/download_request.py index 79240e0..b214659 100644 --- a/models/download_request.py +++ b/models/download_request.py @@ -35,10 +35,10 @@ class DownloadRequest(models.Model): ) data_type = models.CharField( _('Data type'), - default='Well and Monitoring Data', + default=GGMN, choices=( - (WELL_AND_MONITORING_DATA, WELL_AND_MONITORING_DATA), (GGMN, GGMN), + (WELL_AND_MONITORING_DATA, 'Other data'), ), max_length=512 ) diff --git a/tasks/data_file_cache/base_cache.py b/tasks/data_file_cache/base_cache.py index 02cf3b9..f52b25c 100644 --- a/tasks/data_file_cache/base_cache.py +++ b/tasks/data_file_cache/base_cache.py @@ -143,15 +143,19 @@ def merge_data_per_well( os.path.join(well_folder, filename), well_book, ggmn_book, sheetname ) - well_book.active = 0 if ggmn_book: ggmn_book.active = 0 + if well_book: + well_book.active = 0 def merge_data_between_sheets( self, source_file, target_book, target_book_2, sheetname ): """Merge data between sheets""" - if not os.path.exists(source_file) or not target_book: + if ( + not os.path.exists(source_file) + or (not target_book and not target_book_2) + ): return source_file = os.path.join(source_file, sheetname + '.json') data = [] @@ -160,7 +164,9 @@ def merge_data_between_sheets( data = json.loads(_file.read()) # Target book 1 - target_sheet = target_book[sheetname] + target_sheet = None + if target_book: + target_sheet = target_book[sheetname] # Target book 2 target_sheet_2 = None @@ -169,7 +175,8 @@ def merge_data_between_sheets( # Append data from source for row in data: - target_sheet.append(row) + if target_book: + target_sheet.append(row) if target_book_2: target_sheet_2.append(row) @@ -217,7 +224,8 @@ def run(self): # Get data # Well files well_file = self.file_by_type( - self.wells_filename, WELL_AND_MONITORING_DATA) + self.wells_filename, WELL_AND_MONITORING_DATA + ) well_book = load_workbook(well_file) well_ggmn_file = self.file_by_type(self.wells_filename, GGMN) well_ggmn_book = load_workbook(well_ggmn_file) @@ -235,18 +243,21 @@ def run(self): # Save the data wells = self.get_well_queryset() for well in wells: + is_ggmn = well.is_ggmn( + ggmn_organisations_list + ) and well.organisation + print(is_ggmn) + self.merge_data_per_well( - well, self.wells_filename, well_book, - well_ggmn_book if well.is_ggmn( - ggmn_organisations_list - ) and well.organisation else None, + well, self.wells_filename, + well_book if not is_ggmn else None, + well_ggmn_book if is_ggmn else None, ['General Information', 'Hydrogeology', 'Management'] ) self.merge_data_per_well( - well, self.drill_filename, drilling_book, - drilling_ggmn_book if well.is_ggmn( - ggmn_organisations_list - ) and well.organisation else None, + well, self.drill_filename, + drilling_book if not is_ggmn else None, + drilling_ggmn_book if is_ggmn else None, [ SheetName.drilling_and_construction, 'Water Strike', 'Stratigraphic Log', @@ -277,9 +288,13 @@ def run(self): original_ids_found = {} wells = self.get_well_queryset() for well in wells: - if data_type == GGMN and not well.is_ggmn( - ggmn_organisations_list - ): + is_ggmn = well.is_ggmn( + ggmn_organisations_list + ) and well.organisation + + if data_type == GGMN and not is_ggmn: + continue + if data_type == WELL_AND_MONITORING_DATA and is_ggmn: continue if not zip_file: diff --git a/tasks/data_file_cache/country_recache.py b/tasks/data_file_cache/country_recache.py index ea9270f..0df5afa 100644 --- a/tasks/data_file_cache/country_recache.py +++ b/tasks/data_file_cache/country_recache.py @@ -45,9 +45,12 @@ def run(self): ggmn_organisations = [] wells = self.get_well_queryset() for well in wells: - organisations.append(well.organisation.name) if well.is_ggmn(ggmn_organisations_list): ggmn_organisations.append(well.organisation.name) + else: + organisations.append(well.organisation.name) + + # Save the organisation data self.generate_organisations_json_file( WELL_AND_MONITORING_DATA, list(set(organisations)) )