From 52c93c939332da50b0b26b1765f1f58baf149006 Mon Sep 17 00:00:00 2001 From: Felix Kummer Date: Wed, 24 Jul 2024 11:16:17 +0200 Subject: [PATCH] Added new options for higher level processing and visualization. --- conf/test.config | 3 ++ docs/output.md | 4 +- docs/usage.md | 31 +++++++++++++- modules/local/higher_level_force_config.nf | 4 +- nextflow.config | 16 +++++-- nextflow_schema.json | 50 ++++++++++++++++++++++ subworkflows/local/higher_level.nf | 21 ++++++--- workflows/rangeland.nf | 2 +- 8 files changed, 114 insertions(+), 17 deletions(-) diff --git a/conf/test.config b/conf/test.config index f6bd6ee..6b6e679 100644 --- a/conf/test.config +++ b/conf/test.config @@ -52,6 +52,9 @@ params { // Other parameters group_size = 10 + // enable mosaic for result checking + mosaic_visualization = true + validationSchemaIgnoreParams = "peak_yoc_ref,peak_change_ref,herbaceous_yoc_ref,herbaceous_change_ref,woody_yoc_ref,woody_change_ref,config_profile_description,config_profile_name" } diff --git a/docs/output.md b/docs/output.md index 35dd6c5..59dff46 100644 --- a/docs/output.md +++ b/docs/output.md @@ -90,6 +90,8 @@ Next, time series analysis for different vegetation characteristics is performed The resulting trend files can be consulted to view trends for individual tiles. They are saved as symlinks because of their large size. +If the `return_tss` parameter was set to `true`, the pipeline will also output files with the `TSS` ending. These files contain the time series stack(TSS) for the given tile and index or band. Here, for each date of acquisition, an image is available that contains the values for that date. + ### Visualization
@@ -103,7 +105,7 @@ The resulting trend files can be consulted to view trends for individual tiles.
-Two types of common visualizations are generated in the last step of the pipeline. They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing. +Two types of common visualizations are generated in the last step of the pipeline. They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing. Both visualizations are enabled by default but may be disabled in a certain configuration files. Thus, these outputs are optional. ### MultiQC diff --git a/docs/usage.md b/docs/usage.md index 1b53c6e..90aaca1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -4,8 +4,6 @@ > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ -## Introduction - ## Input As most remote sensing workflows, this pipeline relies on numerous sources of data. In the following we will describe the required data and corresponding formats. Mandatory input data consists of satellite data, a digital elevation model, a water vapor database, a data_cube, an area-of-interest specification and an endmember definition. @@ -259,6 +257,35 @@ The group size can be passed using: --group_size '[integer]' ``` +### Higher level processing configuration + +During the higher level processing stage, time series analyses of different satellite bands and indexes is performed. The concrete bands and indexes can be defined using the `indexes` parameter. Spectral unmixing is performed in any case. Thus, passing an empty `indexes` parameter will restrict time series analyses to the results of spectral unmixing. All available indexes can be found [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html) above the `INDEX` parameter. The band/index codes need to be passed in a space-separated string. The default value, `indexes = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2"`, enables time series analyses for the NDVI index and the blue, green, red, near-infrared and both shortwave infrared bands. Note that indexes are usually computed based on certain bands. If these bands are not present in the preprocessed data, these indexes can not be computed. + +The bands and indexes can be passed using: + +```bash +--indexes '[index-string]' +``` + +In so cases, it may be desirable to analyze the the individual images in a time series. To enable such analysis, the parameter `return_tss` can be used. If set to `true`, the pipeline will return time series stacks for each tile and band combination. The option is disabled by default to reduce the output size. + +The time series stack output can be enabled using: + +```bash +--return_tss '[boolean]' +``` + +### Visualization + +The workflow provides two types of results visualization and aggregation. The fine grained mosaic visualization contains all time series analyses results for all tiles in the original resolution. Pyramid visualizations present a broad overview of the same data but at a lower resolution. Both visualizations can be enabled or disabled using the parameters `mosaic_visualization` and `pyramid_visualization`. By default, both visualization methods are enabled. Note that the mosaic visualization is required to be enabled when using the `test` and `test_full` profiles to allow the pipeline to check the correctness of its results (this is the default behavior, make sure to not disable mosaic when using test profiles) . + +The visualizations can be enabled using: + +```bash +mosaic_visualization = '[boolean]' +pyramid_visualization = '[boolean]' +``` + ### FORCE configuration FORCE supports parallel computations. Users can specify the number of threads FORCE can spawn for a single preprocessing, or higher level processing process. This is archived through the `force_threads` parameter. diff --git a/modules/local/higher_level_force_config.nf b/modules/local/higher_level_force_config.nf index 150fdd6..43ffeef 100644 --- a/modules/local/higher_level_force_config.nf +++ b/modules/local/higher_level_force_config.nf @@ -60,8 +60,8 @@ process HIGHER_LEVEL_CONFIG { # spectral index - sed -i "/^INDEX /c\\INDEX = SMA NDVI BLUE GREEN RED NIR SWIR1 SWIR2" \$PARAM - sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM + sed -i "/^INDEX /c\\INDEX = SMA $params.indexes" \$PARAM + ${ params.return_tss ? 'sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM' : '' } # interpolation sed -i "/^INT_DAY /c\\INT_DAY = 8" \$PARAM diff --git a/nextflow.config b/nextflow.config index 9416bad..e561fb5 100644 --- a/nextflow.config +++ b/nextflow.config @@ -34,11 +34,19 @@ params { // FORCE force_threads = 2 + // Higher Level processing configuration + indexes = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2" + return_tss = false + + // Visualization + mosaic_visualization = true + pyramid_visualization = true + // MultiQC options - multiqc_config = null - multiqc_title = null - multiqc_logo = null - max_multiqc_email_size = '25.MB' + multiqc_config = null + multiqc_title = null + multiqc_logo = null + max_multiqc_email_size = '25.MB' multiqc_methods_description = null // Boilerplate options diff --git a/nextflow_schema.json b/nextflow_schema.json index 64e6dfb..60372db 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -131,6 +131,50 @@ }, "fa_icon": "fas fa-satellite" }, + "higher_level_processing_modification": { + "title": "Higher level processing modification", + "type": "object", + "default": "", + "properties": { + "indexes": { + "type": "string", + "default": "NDVI BLUE GREEN RED NIR SWIR1 SWIR2", + "help_text": "Space-separated list of indexes and bands that should be considered in time series analyses. They are indicated by using their established abbreviations. The full list of available indexes is available at https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html under the 'INDEX' parameter. Spectral unmixing is a special index and always activated.", + "description": "Select which bands and indexes should be considered in time series analyses.", + "fa_icon": "fas fa-satellite" + }, + "return_tss": { + "type": "boolean", + "description": "Should the full time series stack be returned.", + "help_text": "This parameter enables or disables the output of the time series stack for each tile and band/index. Here, each band shows the values of a single date.", + "fa_icon": "fas fa-layer-group" + } + }, + "description": "Modify higher level processing behaviour.", + "help_text": "These parameters can be used to modify the behaviour of the time series analyses process in higher level processing.", + "fa_icon": "fas fa-microchip" + }, + "visualization": { + "title": "Visualization", + "type": "object", + "description": "Enable or disable visualizations.", + "default": "", + "properties": { + "mosaic_visualization": { + "type": "boolean", + "default": true, + "description": "Wheter mosaic visualization should be returned.", + "fa_icon": "fas fa-image" + }, + "pyramid_visualization": { + "type": "boolean", + "default": true, + "description": "Whether pyramid visualization should be returned.", + "fa_icon": "far fa-image" + } + }, + "fa_icon": "fas fa-images" + }, "workflow_configuration": { "title": "Workflow configuration", "type": "object", @@ -373,6 +417,12 @@ { "$ref": "#/definitions/remote_sensing_image_options" }, + { + "$ref": "#/definitions/higher_level_processing_modification" + }, + { + "$ref": "#/definitions/visualization" + }, { "$ref": "#/definitions/workflow_configuration" }, diff --git a/subworkflows/local/higher_level.nf b/subworkflows/local/higher_level.nf index ac750df..d1006ac 100644 --- a/subworkflows/local/higher_level.nf +++ b/subworkflows/local/higher_level.nf @@ -31,14 +31,21 @@ workflow HIGHER_LEVEL { trend_files_mosaic = trend_files.groupTuple() // visualizations - FORCE_MOSAIC( trend_files_mosaic, cube_file ) - ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first()) - - FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif') }.map { [ it[1].simpleName.substring(0,11), it[1] ] } .groupTuple() ) - ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first()) + mosaic_files = Channel.empty() + if (params.mosaic_visualization) { + FORCE_MOSAIC( trend_files_mosaic, cube_file ) + mosaic_files = FORCE_MOSAIC.out.trend_files + ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first()) + } + + if (params.pyramid_visualization) { + FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif') }.map { [ it[1].simpleName.substring(0,11), it[1] ] } .groupTuple() ) + ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first()) + } emit: - trend_files = FORCE_MOSAIC.out.trend_files - versions = ch_versions + mosaic_files + + versions = ch_versions } diff --git a/workflows/rangeland.nf b/workflows/rangeland.nf index 57a8f1e..fb44c42 100644 --- a/workflows/rangeland.nf +++ b/workflows/rangeland.nf @@ -128,7 +128,7 @@ workflow RANGELAND { HIGHER_LEVEL( PREPROCESSING.out.tiles_and_masks, cube_file, endmember_file ) ch_versions = ch_versions.mix(HIGHER_LEVEL.out.versions) - grouped_trend_data = HIGHER_LEVEL.out.trend_files.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) + grouped_trend_data = HIGHER_LEVEL.out.mosaic_files.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) // // MODULE: Check results