From 44d890fd7d5ca8e26ccb5b3f3eadc7f5c0f89dc6 Mon Sep 17 00:00:00 2001 From: Jeff Osundwa Date: Mon, 28 Oct 2024 15:09:00 +0300 Subject: [PATCH 1/6] Refactor aggreagtion workflow --- .../workflows/aggregation_workflow_base.py | 123 +++++------------- .../workflows/factor_aggregation_workflow.py | 10 +- .../core/workflows/point_per_cell_workflow.py | 2 +- .../workflows/polyline_per_cell_workflow.py | 2 +- geest/core/workflows/workflow_base.py | 31 ++++- 5 files changed, 70 insertions(+), 98 deletions(-) diff --git a/geest/core/workflows/aggregation_workflow_base.py b/geest/core/workflows/aggregation_workflow_base.py index 8601537..3dd0b7d 100644 --- a/geest/core/workflows/aggregation_workflow_base.py +++ b/geest/core/workflows/aggregation_workflow_base.py @@ -45,6 +45,7 @@ def __init__( self.raster_path_key = ( None # This should be set by the child class e.g. "Indicator Result File" ) + self.aggregation = True def get_weights(self) -> list: """ @@ -57,6 +58,11 @@ def get_weights(self) -> list: weight = layer.get(self.weight_key, 1.0) if weight == "" and len(self.layers) == 1: weight = 1.0 + # Ensure the weight is numeric, cast to float if necessary + try: + weight = float(weight) + except (ValueError, TypeError): + weight = 1.0 # Default fallback to 1.0 if weight is invalid weights.append(weight) return weights @@ -73,7 +79,7 @@ def output_path(self, extension: str) -> str: """ pass - def aggregate(self, input_files: list) -> None: + def aggregate(self, input_files: list) -> str: """ Perform weighted raster aggregation on the found raster files. @@ -108,6 +114,7 @@ def aggregate(self, input_files: list) -> None: # Create QgsRasterCalculatorEntries for each raster layer entries = [] + ref_names = [] for i, raster_layer in enumerate(raster_layers): QgsMessageLog.logMessage( f"Adding raster layer {i+1} to the raster calculator. {raster_layer.source()}", @@ -115,10 +122,13 @@ def aggregate(self, input_files: list) -> None: level=Qgis.Info, ) entry = QgsRasterCalculatorEntry() - entry.ref = f"layer_{i+1}@1" # layer_1@1, layer_2@1, etc. + ref_name = os.path.basename(raster_layer.source()).split(".")[0] + entry.ref = f"{ref_name}_{i+1}@1" # Reference the first band + # entry.ref = f"layer_{i+1}@1" # layer_1@1, layer_2@1, etc. entry.raster = raster_layer entry.bandNumber = 1 entries.append(entry) + ref_names.append(f"{ref_name}_{i+1}") # Assign default weights (you can modify this as needed) weights = self.get_weights() @@ -131,11 +141,11 @@ def aggregate(self, input_files: list) -> None: # Build the calculation expression for weighted average expression = " + ".join( - [f"({weights[i]} * layer_{i+1}@1)" for i in range(layer_count)] + [f"({weights[i]} * {ref_names[i]}@1)" for i in range(layer_count)] ) # Wrap the weighted sum and divide by the sum of weights - expression = f"({expression}) / {sum_weights}" + expression = f"({expression}) / {layer_count}" aggregation_output = self.output_path("tif") QgsMessageLog.logMessage( @@ -176,7 +186,8 @@ def aggregate(self, input_files: list) -> None: converter.convert_to_8bit(aggregation_output, aggregation_output_8bit) if os.path.exists(aggregation_output_8bit): # TODO We should check if developer mode is set and keep the 32-bit raster if it is - os.remove(aggregation_output) + # os.remove(aggregation_output) + pass QgsMessageLog.logMessage( "Raster aggregation completed successfully.", @@ -198,44 +209,6 @@ def aggregate(self, input_files: list) -> None: # That will get passed back to the json model self.attributes[self.result_file_tag] = aggregation_output_8bit - # Fallback sequence to copy QML style - # qml with same name as factor - # qml with generic name of factor.qml - qml_paths = [] - qml_paths.append( - resources_path( - "resources", - "qml", - f"{self.id}.qml", - ) - ) - qml_paths.append( - resources_path( - "resources", - "qml", - f"{self.analysis_mode}.qml", # e.g. factor.qml - ) - ) - qml_dest_path = self.output_path("qml") - for qml_src_path in qml_paths: - if os.path.exists(qml_src_path): - qml_dest_path_8bit = qml_dest_path.replace(".qml", "_8bit.qml") - shutil.copy(qml_src_path, qml_dest_path_8bit) - QgsMessageLog.logMessage( - f"Copied QML style file to {qml_dest_path_8bit}", - tag="Geest", - level=Qgis.Info, - ) - result = aggregated_layer.loadNamedStyle(qml_dest_path_8bit) - if result[0]: # Check if the style was successfully loaded - QgsMessageLog.logMessage( - "Successfully applied QML style.", - tag="Geest", - level=Qgis.Info, - ) - break - - self.context.project().addMapLayer(aggregated_layer) QgsMessageLog.logMessage( "Added raster layer to the map.", tag="Geest", level=Qgis.Info ) @@ -254,10 +227,11 @@ def get_raster_list(self) -> list: for layer in self.layers: path = layer.get(self.raster_path_key, "") - raster_files.append(path) - QgsMessageLog.logMessage( - f"Adding raster: {path}", tag="Geest", level=Qgis.Info - ) + if path: + raster_files.append(path) + QgsMessageLog.logMessage( + f"Adding raster: {path}", tag="Geest", level=Qgis.Info + ) QgsMessageLog.logMessage( f"Total raster files found: {len(raster_files)}", tag="Geest", @@ -265,10 +239,17 @@ def get_raster_list(self) -> list: ) return raster_files - def do_execute(self): + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. """ + _ = current_area # Unused in this analysis + # Log the execution QgsMessageLog.logMessage( f"Executing {self.analysis_mode} Aggregation Workflow", @@ -303,48 +284,8 @@ def do_execute(self): # Perform aggregation only if raster files are provided result_file = self.aggregate(raster_files) - if result_file: - QgsMessageLog.logMessage( - "Aggregation Workflow completed successfully.", - tag="Geest", - level=Qgis.Info, - ) - self.attributes[self.result_file_tag] = result_file - self.attributes["Result"] = ( - f"{self.analysis_mode} Factor Aggregation Workflow Completed" - ) - return True - else: - QgsMessageLog.logMessage( - "Aggregation failed due to missing or invalid raster files.", - tag="Geest", - level=Qgis.Warning, - ) - self.attributes[self.result_file_tag] = None - self.attributes["Result"] = ( - f"{self.analysis_mode} Aggregation Workflow Failed" - ) - return False - def _process_features_for_area(self): - pass + return result_file - # Default implementation of the abstract method - not used in this workflow - def _process_raster_for_area( - self, - current_area: QgsGeometry, - current_bbox: QgsGeometry, - area_raster: str, - index: int, - ): - """ - Executes the actual workflow logic for a single area using a raster. - - :current_area: Current polygon from our study area. - :current_bbox: Bounding box of the above area. - :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. - :index: Index of the current area. - - :return: Path to the reclassified raster. - """ - pass + def do_execute(self): + self._execute() diff --git a/geest/core/workflows/factor_aggregation_workflow.py b/geest/core/workflows/factor_aggregation_workflow.py index 767503b..4facef2 100644 --- a/geest/core/workflows/factor_aggregation_workflow.py +++ b/geest/core/workflows/factor_aggregation_workflow.py @@ -31,6 +31,8 @@ def __init__( self.weight_key = "Indicator Weighting" self.result_file_tag = "Factor Result File" self.raster_path_key = "Indicator Result File" + self.workflow_is_legacy = False + self.layer_id = self.id def output_path(self, extension: str) -> str: """ @@ -45,8 +47,6 @@ def output_path(self, extension: str) -> str: """ directory = os.path.join( self.workflow_directory, - self.aggregation_attributes.get("Dimension ID").lower().replace(" ", "_"), - self.aggregation_attributes.get("Factor ID").lower().replace(" ", "_"), ) # Create the directory if it doesn't exist if not os.path.exists(directory): @@ -59,3 +59,9 @@ def output_path(self, extension: str) -> str: def _process_features_for_area(self): pass + + def _process_raster_for_area(self): + pass + + def do_execute(self): + self._execute() diff --git a/geest/core/workflows/point_per_cell_workflow.py b/geest/core/workflows/point_per_cell_workflow.py index b6b2bb3..e26f44f 100644 --- a/geest/core/workflows/point_per_cell_workflow.py +++ b/geest/core/workflows/point_per_cell_workflow.py @@ -94,7 +94,7 @@ def _process_features_for_area( current_bbox, index, value_field="value", - default_value=255, + default_value=0, ) return raster_output diff --git a/geest/core/workflows/polyline_per_cell_workflow.py b/geest/core/workflows/polyline_per_cell_workflow.py index 9764f1c..65e3428 100644 --- a/geest/core/workflows/polyline_per_cell_workflow.py +++ b/geest/core/workflows/polyline_per_cell_workflow.py @@ -96,7 +96,7 @@ def _process_features_for_area( current_bbox, index, value_field="value", - default_value=255, + default_value=0, ) return raster_output diff --git a/geest/core/workflows/workflow_base.py b/geest/core/workflows/workflow_base.py index 313733e..b1041b3 100644 --- a/geest/core/workflows/workflow_base.py +++ b/geest/core/workflows/workflow_base.py @@ -80,6 +80,7 @@ def __init__( self.layer_id = self.attributes.get("ID", "").lower().replace(" ", "_") self.attributes["Result"] = "Not Run" self.workflow_is_legacy = True + self.aggregation = False # # Every concrete subclass needs to implement these three methods @@ -136,6 +137,24 @@ def _process_raster_for_area( """ pass + @abstractmethod + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the actual workflow logic for a single area using an aggregate. + + :current_area: Current polygon from our study area. + :current_bbox: Bounding box of the above area. + :index: Index of the current area. + + :return: Path to the reclassified raster. + """ + pass + # ------------------- END OF ABSCRACT METHODS ------------------- def execute(self) -> bool: @@ -221,9 +240,9 @@ def execute(self) -> bool: area_features=area_features, index=index, ) - - else: # assumes we are processing a raster input - + elif ( + self.aggregate == False + ): # assumes we are processing a raster input area_raster = self._subset_raster_layer( bbox=current_bbox, index=index ) @@ -233,6 +252,12 @@ def execute(self) -> bool: area_raster=area_raster, index=index, ) + if self.aggregation == True: # we are processing an aggregate + raster_output = self._process_aggregate_for_area( + current_area=current_area, + current_bbox=current_bbox, + index=index, + ) # Multiply the area by its matching mask layer in study_area folder masked_layer = self._mask_raster( From 6bde0ddd38c73c58550bb9b891bdc01bc1abcf10 Mon Sep 17 00:00:00 2001 From: Tim Sutton Date: Mon, 28 Oct 2024 14:47:42 +0000 Subject: [PATCH 2/6] Update aggregator to work on individual parts rather than vrt --- geest/core/json_tree_item.py | 9 +++-- .../workflows/aggregation_workflow_base.py | 40 ++++++++----------- geest/gui/views/treeview.py | 6 +-- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/geest/core/json_tree_item.py b/geest/core/json_tree_item.py index 2b28829..3f833e3 100644 --- a/geest/core/json_tree_item.py +++ b/geest/core/json_tree_item.py @@ -285,7 +285,8 @@ def getFactorAttributes(self): attributes["Factor ID"] = self.data(0) attributes["Indicators"] = [ { - "Indicator ID": i, + "Indicator No": i, + "Indicator ID": child.data(3).get("ID", ""), "Indicator Name": child.data(0), "Indicator Weighting": child.data(2), "Indicator Result File": child.data(3).get( @@ -304,7 +305,8 @@ def getDimensionAttributes(self): attributes["Dimension ID"] = self.data(0) attributes["Factors"] = [ { - "Factor ID": i, + "Factor No": i, + "Factor ID": child.data(3).get("ID", ""), "Factor Name": child.data(0), "Factor Weighting": child.data(2), "Factor Result File": child.data(3).get(f"Factor Result File", ""), @@ -325,7 +327,8 @@ def getAnalysisAttributes(self): attributes["Dimensions"] = [ { - "Dimension ID": i, + "Dimension No": i, + "Dimension ID": child.data(3).get("id", ""), "Dimension Name": child.data(0), "Dimension Weighting": child.data(2), "Dimension Result File": child.data(3).get( diff --git a/geest/core/workflows/aggregation_workflow_base.py b/geest/core/workflows/aggregation_workflow_base.py index 3dd0b7d..f7dc054 100644 --- a/geest/core/workflows/aggregation_workflow_base.py +++ b/geest/core/workflows/aggregation_workflow_base.py @@ -33,7 +33,6 @@ def __init__( super().__init__( item, feedback, context ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree - self.attributes = item.data(3) self.aggregation_attributes = None # This should be set by the child class e.g. item.getIndicatorAttributes() self.analysis_mode = self.attributes.get("Analysis Mode", "") self.id = None # This should be set by the child class @@ -46,6 +45,7 @@ def __init__( None # This should be set by the child class e.g. "Indicator Result File" ) self.aggregation = True + self.workflow_is_legacy = False def get_weights(self) -> list: """ @@ -66,24 +66,12 @@ def get_weights(self) -> list: weights.append(weight) return weights - def output_path(self, extension: str) -> str: - """ - Define output path for the aggregated raster based on the analysis mode. - - Parameters: - extension (str): The file extension for the output file. - - Returns: - str: Path to the aggregated raster file. - - """ - pass - - def aggregate(self, input_files: list) -> str: + def aggregate(self, input_files: list, index: int) -> str: """ Perform weighted raster aggregation on the found raster files. :param input_files: List of raster file paths to aggregate. + :param index: The index of the area being processed. :return: Path to the aggregated raster file. """ @@ -147,7 +135,9 @@ def aggregate(self, input_files: list) -> str: # Wrap the weighted sum and divide by the sum of weights expression = f"({expression}) / {layer_count}" - aggregation_output = self.output_path("tif") + aggregation_output = os.path.join( + self.workflow_directory, f"{self.layer_id}_aggregated_{index}.tif" + ) QgsMessageLog.logMessage( f"Aggregating {len(input_files)} raster layers to {aggregation_output}", tag="Geest", @@ -209,24 +199,28 @@ def aggregate(self, input_files: list) -> str: # That will get passed back to the json model self.attributes[self.result_file_tag] = aggregation_output_8bit - QgsMessageLog.logMessage( - "Added raster layer to the map.", tag="Geest", level=Qgis.Info - ) return aggregation_output_8bit - def get_raster_list(self) -> list: + def get_raster_list(self, index) -> list: """ Get the list of rasters from the attributes that will be aggregated. (Factor Aggregation, Dimension Aggregation, Analysis). + Parameters: + index (int): The index of the area being processed. + Returns: list: List of found raster file paths. """ raster_files = [] for layer in self.layers: - path = layer.get(self.raster_path_key, "") + id = layer.get("Indicator ID", "").lower() + layer_folder = os.path.dirname(layer.get("Indicator Result File", "")) + path = os.path.join( + self.workflow_directory, layer_folder, f"{id}_masked_{index}.tif" + ) if path: raster_files.append(path) QgsMessageLog.logMessage( @@ -263,7 +257,7 @@ def _process_aggregate_for_area( level=Qgis.Info, ) - raster_files = self.get_raster_list() + raster_files = self.get_raster_list(index) if not raster_files or not isinstance(raster_files, list): QgsMessageLog.logMessage( @@ -283,7 +277,7 @@ def _process_aggregate_for_area( ) # Perform aggregation only if raster files are provided - result_file = self.aggregate(raster_files) + result_file = self.aggregate(raster_files, index) return result_file diff --git a/geest/gui/views/treeview.py b/geest/gui/views/treeview.py index 0a82d1c..bf3defc 100644 --- a/geest/gui/views/treeview.py +++ b/geest/gui/views/treeview.py @@ -239,7 +239,7 @@ def data(self, index, role): return item.getIcon() elif ( role == Qt.DecorationRole and index.column() == 1 - ): # Icon for the status columen + ): # Icon for the status column return item.getStatusIcon() elif role == Qt.ToolTipRole and index.column() == 1: return item.getStatusTooltip() @@ -266,7 +266,7 @@ def setData(self, index, value, role=Qt.EditRole): item = index.internalPointer() column = index.column() - if column == 2: # Weighting column + if column == 2: # tree_view column try: value = float(value) return item.setData(column, f"{value:.2f}") @@ -428,7 +428,7 @@ def auto_assign_layer_weightings(self, factor_item): layer_item.setData(2, f"{layer_weighting:.2f}") # Update the factor's total weighting factor_item.setData(2, "1.00") - self.update_font_color(factor_item, QColor(Qt.green)) + # self.update_font_color(factor_item, QColor(Qt.green)) self.layoutChanged.emit() def add_factor(self, dimension_item): From 0e73e5e39edca8ac1cbc3ff0271e52a5736430cc Mon Sep 17 00:00:00 2001 From: Tim Sutton Date: Mon, 28 Oct 2024 17:30:13 +0000 Subject: [PATCH 3/6] Make all concreate classes adhere to ABC for workflows --- geest/core/workflows/acled_impact_workflow.py | 11 ++++++ .../workflows/aggregation_workflow_base.py | 38 +++++-------------- .../analysis_aggregation_workflow.py | 24 ------------ .../workflows/default_index_score_workflow.py | 11 ++++++ .../dimension_aggregation_workflow.py | 24 ------------ geest/core/workflows/dont_use_workflow.py | 11 ++++++ .../workflows/factor_aggregation_workflow.py | 32 ---------------- .../multi_buffer_distances_workflow.py | 11 ++++++ .../core/workflows/point_per_cell_workflow.py | 11 ++++++ .../workflows/polygon_per_cell_workflow.py | 11 ++++++ .../workflows/polyline_per_cell_workflow.py | 11 ++++++ geest/core/workflows/raster_layer_workflow.py | 11 ++++++ .../raster_reclassification_workflow.py | 11 ++++++ .../core/workflows/safety_polygon_workflow.py | 11 ++++++ .../core/workflows/safety_raster_workflow.py | 11 ++++++ .../workflows/single_point_buffer_workflow.py | 11 ++++++ 16 files changed, 142 insertions(+), 108 deletions(-) diff --git a/geest/core/workflows/acled_impact_workflow.py b/geest/core/workflows/acled_impact_workflow.py index 0d66644..74d9666 100644 --- a/geest/core/workflows/acled_impact_workflow.py +++ b/geest/core/workflows/acled_impact_workflow.py @@ -393,3 +393,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/aggregation_workflow_base.py b/geest/core/workflows/aggregation_workflow_base.py index f7dc054..7870c54 100644 --- a/geest/core/workflows/aggregation_workflow_base.py +++ b/geest/core/workflows/aggregation_workflow_base.py @@ -170,36 +170,11 @@ def aggregate(self, input_files: list, index: int) -> str: ) return None - converter = RasterConverter() - aggregation_output_8bit = aggregation_output.replace(".tif", "_8bit.tif") - # Convert the aggregated raster to 8-bit - converter.convert_to_8bit(aggregation_output, aggregation_output_8bit) - if os.path.exists(aggregation_output_8bit): - # TODO We should check if developer mode is set and keep the 32-bit raster if it is - # os.remove(aggregation_output) - pass - - QgsMessageLog.logMessage( - "Raster aggregation completed successfully.", - tag="Geest", - level=Qgis.Info, - ) - # Add the aggregated raster to the map - aggregated_layer = QgsRasterLayer( - aggregation_output_8bit, f"aggregated_{self.id}.tif" - ) - if not aggregated_layer.isValid(): - QgsMessageLog.logMessage( - "Aggregate layer is not valid.", - tag="Geest", - level=Qgis.Critical, - ) - return None # WRite the output path to the attributes # That will get passed back to the json model - self.attributes[self.result_file_tag] = aggregation_output_8bit + self.attributes[self.result_file_tag] = aggregation_output - return aggregation_output_8bit + return aggregation_output def get_raster_list(self, index) -> list: """ @@ -243,6 +218,7 @@ def _process_aggregate_for_area( Executes the workflow, reporting progress through the feedback object and checking for cancellation. """ _ = current_area # Unused in this analysis + _ = current_bbox # Unused in this analysis # Log the execution QgsMessageLog.logMessage( @@ -281,5 +257,11 @@ def _process_aggregate_for_area( return result_file + def _process_features_for_area(self): + pass + + def _process_raster_for_area(self): + pass + def do_execute(self): - self._execute() + pass diff --git a/geest/core/workflows/analysis_aggregation_workflow.py b/geest/core/workflows/analysis_aggregation_workflow.py index e4f9b5f..86eaacc 100644 --- a/geest/core/workflows/analysis_aggregation_workflow.py +++ b/geest/core/workflows/analysis_aggregation_workflow.py @@ -31,27 +31,3 @@ def __init__( self.weight_key = "Dimension Weighting" self.result_file_tag = "Analysis Result File" self.raster_path_key = "Dimension Result File" - - def output_path(self, extension: str) -> str: - """ - Define output path for the aggregated raster based on the analysis mode. - - Parameters: - extension (str): The file extension for the output file. - - Returns: - str: Path to the aggregated raster file. - - """ - directory = self.workflow_directory - # Create the directory if it doesn't exist - if not os.path.exists(directory): - os.makedirs(directory) - - return os.path.join( - directory, - f"aggregate_{self.id}" + f".{extension}", - ) - - def _process_features_for_area(self): - pass diff --git a/geest/core/workflows/default_index_score_workflow.py b/geest/core/workflows/default_index_score_workflow.py index ee6c72f..1fea1e1 100644 --- a/geest/core/workflows/default_index_score_workflow.py +++ b/geest/core/workflows/default_index_score_workflow.py @@ -156,3 +156,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/dimension_aggregation_workflow.py b/geest/core/workflows/dimension_aggregation_workflow.py index 375a174..0f61cb6 100644 --- a/geest/core/workflows/dimension_aggregation_workflow.py +++ b/geest/core/workflows/dimension_aggregation_workflow.py @@ -32,27 +32,3 @@ def __init__( self.weight_key = "Factor Weighting" self.result_file_tag = "Dimension Result File" self.raster_path_key = "Factor Result File" - - def output_path(self, extension: str) -> str: - """ - Define output path for the aggregated raster based on the analysis mode. - - Parameters: - extension (str): The file extension for the output file. - - Returns: - str: Path to the aggregated raster file. - - """ - directory = os.path.join(self.workflow_directory, self.id) - # Create the directory if it doesn't exist - if not os.path.exists(directory): - os.makedirs(directory) - - return os.path.join( - directory, - f"aggregate_{self.id}" + f".{extension}", - ) - - def _process_features_for_area(self): - pass diff --git a/geest/core/workflows/dont_use_workflow.py b/geest/core/workflows/dont_use_workflow.py index a161e6a..e88a7e5 100644 --- a/geest/core/workflows/dont_use_workflow.py +++ b/geest/core/workflows/dont_use_workflow.py @@ -57,3 +57,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/factor_aggregation_workflow.py b/geest/core/workflows/factor_aggregation_workflow.py index 4facef2..3bf273d 100644 --- a/geest/core/workflows/factor_aggregation_workflow.py +++ b/geest/core/workflows/factor_aggregation_workflow.py @@ -33,35 +33,3 @@ def __init__( self.raster_path_key = "Indicator Result File" self.workflow_is_legacy = False self.layer_id = self.id - - def output_path(self, extension: str) -> str: - """ - Define output path for the aggregated raster based on the analysis mode. - - Parameters: - extension (str): The file extension for the output file. - - Returns: - str: Path to the aggregated raster file. - - """ - directory = os.path.join( - self.workflow_directory, - ) - # Create the directory if it doesn't exist - if not os.path.exists(directory): - os.makedirs(directory) - - return os.path.join( - directory, - f"aggregate_{self.id}" + f".{extension}", - ) - - def _process_features_for_area(self): - pass - - def _process_raster_for_area(self): - pass - - def do_execute(self): - self._execute() diff --git a/geest/core/workflows/multi_buffer_distances_workflow.py b/geest/core/workflows/multi_buffer_distances_workflow.py index aa1ead1..c9d7be3 100644 --- a/geest/core/workflows/multi_buffer_distances_workflow.py +++ b/geest/core/workflows/multi_buffer_distances_workflow.py @@ -623,3 +623,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/point_per_cell_workflow.py b/geest/core/workflows/point_per_cell_workflow.py index e26f44f..5a191bd 100644 --- a/geest/core/workflows/point_per_cell_workflow.py +++ b/geest/core/workflows/point_per_cell_workflow.py @@ -124,3 +124,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/polygon_per_cell_workflow.py b/geest/core/workflows/polygon_per_cell_workflow.py index 5ef8fef..ad82039 100644 --- a/geest/core/workflows/polygon_per_cell_workflow.py +++ b/geest/core/workflows/polygon_per_cell_workflow.py @@ -121,3 +121,14 @@ def do_execute(self): Execute the workflow. """ self._execute() + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/polyline_per_cell_workflow.py b/geest/core/workflows/polyline_per_cell_workflow.py index 65e3428..3739357 100644 --- a/geest/core/workflows/polyline_per_cell_workflow.py +++ b/geest/core/workflows/polyline_per_cell_workflow.py @@ -126,3 +126,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/raster_layer_workflow.py b/geest/core/workflows/raster_layer_workflow.py index b801632..49ca5ad 100644 --- a/geest/core/workflows/raster_layer_workflow.py +++ b/geest/core/workflows/raster_layer_workflow.py @@ -80,3 +80,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/raster_reclassification_workflow.py b/geest/core/workflows/raster_reclassification_workflow.py index 3754f78..718ccd3 100644 --- a/geest/core/workflows/raster_reclassification_workflow.py +++ b/geest/core/workflows/raster_reclassification_workflow.py @@ -279,3 +279,14 @@ def _process_features_for_area( :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/safety_polygon_workflow.py b/geest/core/workflows/safety_polygon_workflow.py index 8b50d89..35a0b3b 100644 --- a/geest/core/workflows/safety_polygon_workflow.py +++ b/geest/core/workflows/safety_polygon_workflow.py @@ -142,6 +142,17 @@ def _process_raster_for_area( """ pass + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass + # TODO Remove when all workflows are refactored def do_execute(self): """ diff --git a/geest/core/workflows/safety_raster_workflow.py b/geest/core/workflows/safety_raster_workflow.py index 4a10f0f..aa335b3 100644 --- a/geest/core/workflows/safety_raster_workflow.py +++ b/geest/core/workflows/safety_raster_workflow.py @@ -286,3 +286,14 @@ def _process_features_for_area( :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass diff --git a/geest/core/workflows/single_point_buffer_workflow.py b/geest/core/workflows/single_point_buffer_workflow.py index e24f31d..2153f83 100644 --- a/geest/core/workflows/single_point_buffer_workflow.py +++ b/geest/core/workflows/single_point_buffer_workflow.py @@ -179,3 +179,14 @@ def _process_raster_for_area( :return: Path to the reclassified raster. """ pass + + def _process_aggregate_for_area( + self, + current_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ): + """ + Executes the workflow, reporting progress through the feedback object and checking for cancellation. + """ + pass From b9ff27fd3dae6c993d617d95c39e380979acdc5e Mon Sep 17 00:00:00 2001 From: Tim Sutton Date: Mon, 28 Oct 2024 17:48:11 +0000 Subject: [PATCH 4/6] Fixes for aggregration - make masks work --- geest/core/workflows/workflow_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geest/core/workflows/workflow_base.py b/geest/core/workflows/workflow_base.py index 7c598aa..12423b7 100644 --- a/geest/core/workflows/workflow_base.py +++ b/geest/core/workflows/workflow_base.py @@ -254,7 +254,7 @@ def execute(self) -> bool: area_raster=area_raster, index=index, ) - if self.aggregation == True: # we are processing an aggregate + elif self.aggregation == True: # we are processing an aggregate raster_output = self._process_aggregate_for_area( current_area=current_area, current_bbox=current_bbox, @@ -573,7 +573,7 @@ def _mask_raster( "SOURCE_CRS": None, "TARGET_CRS": None, "TARGET_EXTENT": None, - "NODATA": None, + "NODATA": 255, "ALPHA_BAND": False, "CROP_TO_CUTLINE": True, "KEEP_RESOLUTION": False, @@ -582,7 +582,7 @@ def _mask_raster( "Y_RESOLUTION": None, "MULTITHREADING": False, "OPTIONS": "", - "DATA_TYPE": 0, + "DATA_TYPE": 0, # byte - TODO softcode this for aggregation we want float "EXTRA": "", } processing.run("gdal:cliprasterbymasklayer", params) From 4379b160ef2cadf5227ce0b0db2344339ef671bb Mon Sep 17 00:00:00 2001 From: Tim Sutton Date: Mon, 28 Oct 2024 18:03:13 +0000 Subject: [PATCH 5/6] Add layers to a layer group when completed and refresh layers rather than readding if they already exist in QGIS project. --- geest/gui/panels/tree_panel.py | 55 ++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/geest/gui/panels/tree_panel.py b/geest/gui/panels/tree_panel.py index 22b0a1f..5e3e90d 100644 --- a/geest/gui/panels/tree_panel.py +++ b/geest/gui/panels/tree_panel.py @@ -25,7 +25,14 @@ QTableWidgetItem, QPushButton, ) -from qgis.core import QgsMessageLog, Qgis, QgsRasterLayer, QgsProject, QgsVectorLayer +from qgis.core import ( + QgsMessageLog, + Qgis, + QgsRasterLayer, + QgsProject, + QgsVectorLayer, + QgsLayerTreeGroup, +) from functools import partial from geest.gui.views import JsonTreeView, JsonTreeModel from geest.utilities import resources_path @@ -806,8 +813,50 @@ def on_workflow_completed(self, item, success): output_file = item.data(3).get("Indicator Result File", None) if output_file: - layer = QgsRasterLayer(output_file, item.data(0)) - QgsProject.instance().addMapLayer(layer) + layer_name = item.data(0) + layer = QgsRasterLayer(output_file, layer_name) + + if not layer.isValid(): + QgsMessageLog.logMessage( + f"Layer {layer_name} is invalid and cannot be added.", + tag="Geest", + level=Qgis.Warning, + ) + return + + project = QgsProject.instance() + + # Check if 'Geest' group exists, otherwise create it + geest_group = project.layerTreeRoot().findGroup("Geest") + if geest_group is None: + geest_group = project.layerTreeRoot().addGroup("Geest") + + # Check if a layer with the same data source exists in the 'Geest' group + existing_layer = None + for layer_id in QgsProject.instance().mapLayers().keys(): + current_layer = QgsProject.instance().mapLayer(layer_id) + if current_layer.source() == output_file and geest_group.findLayer( + current_layer.id() + ): + existing_layer = current_layer + break + + # If the layer exists, refresh it instead of removing and re-adding + if existing_layer is not None: + QgsMessageLog.logMessage( + f"Refreshing existing layer: {existing_layer.name()}", + tag="Geest", + level=Qgis.Info, + ) + existing_layer.reload() + else: + # Add the new layer to the 'Geest' group + QgsProject.instance().addMapLayer(layer, False) + geest_group.addLayer(layer) + QgsMessageLog.logMessage( + f"Added layer: {layer.name()}", tag="Geest", level=Qgis.Info + ) + item.updateStatus() self.save_json_to_working_directory() From a8cc86de6bb83c798196e3eb9be3bc69b7d7ad21 Mon Sep 17 00:00:00 2001 From: Tim Sutton Date: Mon, 28 Oct 2024 18:34:03 +0000 Subject: [PATCH 6/6] Add layers into TOC in same tree structure as layers --- geest/gui/panels/tree_panel.py | 119 ++++++++++++++++----------------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/geest/gui/panels/tree_panel.py b/geest/gui/panels/tree_panel.py index 5e3e90d..a0306dd 100644 --- a/geest/gui/panels/tree_panel.py +++ b/geest/gui/panels/tree_panel.py @@ -595,20 +595,63 @@ def add_to_map(self, item): layer_uri = item.data(3).get( f"{item.role.title()} Result File" ) # title = title case string - layer_name = item.data(0) - QgsMessageLog.logMessage( - f"Adding {layer_uri} to the map.", tag="Geest", level=Qgis.Info - ) - # Add the aggregated raster to the map - layer = QgsRasterLayer(layer_uri, layer_name) - if layer.isValid(): - QgsProject.instance().addMapLayer(layer) - else: - QgsMessageLog.logMessage( - f"Failed to add the {layer_name} raster to the map.", - tag="Geest", - level=Qgis.Critical, - ) + if layer_uri: + layer_name = item.data(0) + layer = QgsRasterLayer(layer_uri, layer_name) + + if not layer.isValid(): + QgsMessageLog.logMessage( + f"Layer {layer_name} is invalid and cannot be added.", + tag="Geest", + level=Qgis.Warning, + ) + return + + project = QgsProject.instance() + + # Check if 'Geest' group exists, otherwise create it + root = project.layerTreeRoot() + geest_group = root.findGroup("Geest") + if geest_group is None: + geest_group = root.insertGroup( + 0, "Geest" + ) # Insert at the top of the layers panel + + # Traverse the tree view structure to determine the appropriate subgroup based on paths + path_list = item.getPaths() + parent_group = geest_group + for path in path_list: + sub_group = parent_group.findGroup(path) + if sub_group is None: + sub_group = parent_group.addGroup(path) + parent_group = sub_group + + # Check if a layer with the same data source exists in the correct group + existing_layer = None + for child in parent_group.children(): + if isinstance(child, QgsLayerTreeGroup): + continue + if child.layer().source() == layer_uri: + existing_layer = child.layer() + break + + # If the layer exists, refresh it instead of removing and re-adding + if existing_layer is not None: + QgsMessageLog.logMessage( + f"Refreshing existing layer: {existing_layer.name()}", + tag="Geest", + level=Qgis.Info, + ) + existing_layer.reload() + else: + # Add the new layer to the appropriate subgroup + QgsProject.instance().addMapLayer(layer, False) + parent_group.addLayer(layer) + QgsMessageLog.logMessage( + f"Added layer: {layer.name()} to group: {parent_group.name()}", + tag="Geest", + level=Qgis.Info, + ) def edit_dimension_aggregation(self, dimension_item): """Open the DimensionAggregationDialog for editing the weightings of factors in a dimension.""" @@ -811,55 +854,11 @@ def on_workflow_completed(self, item, success): self.overall_progress_bar.setValue(self.overall_progress_bar.value() + 1) self.workflow_progress_bar.setValue(100) - output_file = item.data(3).get("Indicator Result File", None) - if output_file: - layer_name = item.data(0) - layer = QgsRasterLayer(output_file, layer_name) - - if not layer.isValid(): - QgsMessageLog.logMessage( - f"Layer {layer_name} is invalid and cannot be added.", - tag="Geest", - level=Qgis.Warning, - ) - return - - project = QgsProject.instance() - - # Check if 'Geest' group exists, otherwise create it - geest_group = project.layerTreeRoot().findGroup("Geest") - if geest_group is None: - geest_group = project.layerTreeRoot().addGroup("Geest") - - # Check if a layer with the same data source exists in the 'Geest' group - existing_layer = None - for layer_id in QgsProject.instance().mapLayers().keys(): - current_layer = QgsProject.instance().mapLayer(layer_id) - if current_layer.source() == output_file and geest_group.findLayer( - current_layer.id() - ): - existing_layer = current_layer - break - - # If the layer exists, refresh it instead of removing and re-adding - if existing_layer is not None: - QgsMessageLog.logMessage( - f"Refreshing existing layer: {existing_layer.name()}", - tag="Geest", - level=Qgis.Info, - ) - existing_layer.reload() - else: - # Add the new layer to the 'Geest' group - QgsProject.instance().addMapLayer(layer, False) - geest_group.addLayer(layer) - QgsMessageLog.logMessage( - f"Added layer: {layer.name()}", tag="Geest", level=Qgis.Info - ) - item.updateStatus() self.save_json_to_working_directory() + self.add_to_map(item) + # Now cancel the animated icon node_index = self.model.itemIndex(item)