Added new options for higher level processing and visualization.

conf/test.config

@@ -52,6 +52,9 @@ params {
     // Other parameters
     group_size = 10
 
+    // enable mosaic for result checking
+    mosaic_visualization  = true
+
     validationSchemaIgnoreParams = "peak_yoc_ref,peak_change_ref,herbaceous_yoc_ref,herbaceous_change_ref,woody_yoc_ref,woody_change_ref,config_profile_description,config_profile_name"
 }
 

docs/output.md

@@ -90,6 +90,8 @@ Next, time series analysis for different vegetation characteristics is performed
 
 The resulting trend files can be consulted to view trends for individual tiles. They are saved as symlinks because of their large size.
 
+If the `return_tss` parameter was set to `true`, the pipeline will also output files with the `TSS` ending. These files contain the time series stack(TSS) for the given tile and index or band. Here, for each date of acquisition, an image is available that contains the values for that date.
+
 ### Visualization
 
 <details markdown="1">
@@ -103,7 +105,7 @@ The resulting trend files can be consulted to view trends for individual tiles.
 
 </details>
 
-Two types of common visualizations are generated in the last step of the pipeline. They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing.
+Two types of common visualizations are generated in the last step of the pipeline. They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing. Both visualizations are enabled by default but may be disabled in a certain configuration files. Thus, these outputs are optional.
 
 ### MultiQC
 

docs/usage.md

@@ -4,8 +4,6 @@
 
 > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._
 
-## Introduction
-
 ## Input
 
 As most remote sensing workflows, this pipeline relies on numerous sources of data. In the following we will describe the required data and corresponding formats. Mandatory input data consists of satellite data, a digital elevation model, a water vapor database, a data_cube, an area-of-interest specification and an endmember definition.
@@ -259,6 +257,35 @@ The group size can be passed using:
 --group_size '[integer]'
 ```
 
+### Higher level processing configuration
+
+During the higher level processing stage, time series analyses of different satellite bands and indexes is performed. The concrete bands and indexes can be defined using the `indexes` parameter. Spectral unmixing is performed in any case. Thus, passing an empty `indexes` parameter will restrict time series analyses to the results of spectral unmixing. All available indexes can be found [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html) above the `INDEX` parameter. The band/index codes need to be passed in a space-separated string. The default value, `indexes = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2"`, enables time series analyses for the NDVI index and the blue, green, red, near-infrared and both shortwave infrared bands. Note that indexes are usually computed based on certain bands. If these bands are not present in the preprocessed data, these indexes can not be computed.
+
+The bands and indexes can be passed using:
+
+```bash
+--indexes '[index-string]'
+```
+
+In so cases, it may be desirable to analyze the the individual images in a time series. To enable such analysis, the parameter `return_tss` can be used. If set to `true`, the pipeline will return time series stacks for each tile and band combination. The option is disabled by default to reduce the output size.
+
+The time series stack output can be enabled using:
+
+```bash
+--return_tss '[boolean]'
+```
+
+### Visualization
+
+The workflow provides two types of results visualization and aggregation. The fine grained mosaic visualization contains all time series analyses results for all tiles in the original resolution. Pyramid visualizations present a broad overview of the same data but at a lower resolution. Both visualizations can be enabled or disabled using the parameters `mosaic_visualization` and `pyramid_visualization`. By default, both visualization methods are enabled. Note that the mosaic visualization is required to be enabled when using the `test` and `test_full` profiles to allow the pipeline to check the correctness of its results (this is the default behavior, make sure to not disable mosaic when using test profiles) .
+
+The visualizations can be enabled using:
+
+```bash
+mosaic_visualization  = '[boolean]'
+pyramid_visualization = '[boolean]'
+```
+
 ### FORCE configuration
 
 FORCE supports parallel computations. Users can specify the number of threads FORCE can spawn for a single preprocessing, or higher level processing process. This is archived through the `force_threads` parameter.

modules/local/higher_level_force_config.nf

@@ -60,8 +60,8 @@ process HIGHER_LEVEL_CONFIG {
 
 
     # spectral index
-    sed -i "/^INDEX /c\\INDEX = SMA NDVI BLUE GREEN RED NIR SWIR1 SWIR2" \$PARAM
-    sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM
+    sed -i "/^INDEX /c\\INDEX = SMA $params.indexes" \$PARAM
+    ${ params.return_tss ? 'sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM' : '' }
 
     # interpolation
     sed -i "/^INT_DAY /c\\INT_DAY = 8" \$PARAM

nextflow.config

@@ -34,11 +34,19 @@ params {
     // FORCE
     force_threads = 2
 
+    // Higher Level processing configuration
+    indexes    = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2"
+    return_tss = false
+
+    // Visualization
+    mosaic_visualization  = true
+    pyramid_visualization = true
+
     // MultiQC options
-    multiqc_config             = null
-    multiqc_title              = null
-    multiqc_logo               = null
-    max_multiqc_email_size     = '25.MB'
+    multiqc_config              = null
+    multiqc_title               = null
+    multiqc_logo                = null
+    max_multiqc_email_size      = '25.MB'
     multiqc_methods_description = null
 
     // Boilerplate options

nextflow_schema.json

@@ -131,6 +131,50 @@
             },
             "fa_icon": "fas fa-satellite"
         },
+        "higher_level_processing_modification": {
+            "title": "Higher level processing modification",
+            "type": "object",
+            "default": "",
+            "properties": {
+                "indexes": {
+                    "type": "string",
+                    "default": "NDVI BLUE GREEN RED NIR SWIR1 SWIR2",
+                    "help_text": "Space-separated list of indexes and bands that should be considered in time series analyses. They are indicated by using their established abbreviations. The full list of available indexes is available at https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html under the 'INDEX' parameter. Spectral unmixing is a special index and always activated.",
+                    "description": "Select which bands and indexes should be considered in time series analyses.",
+                    "fa_icon": "fas fa-satellite"
+                },
+                "return_tss": {
+                    "type": "boolean",
+                    "description": "Should the full time series stack be returned.",
+                    "help_text": "This parameter enables or disables the output of the time series stack for each tile and band/index. Here, each band shows the values of a single date.",
+                    "fa_icon": "fas fa-layer-group"
+                }
+            },
+            "description": "Modify higher level processing behaviour.",
+            "help_text": "These parameters can be used to modify the behaviour of the time series analyses process in higher level processing.",
+            "fa_icon": "fas fa-microchip"
+        },
+        "visualization": {
+            "title": "Visualization",
+            "type": "object",
+            "description": "Enable or disable visualizations.",
+            "default": "",
+            "properties": {
+                "mosaic_visualization": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Wheter mosaic visualization should be returned.",
+                    "fa_icon": "fas fa-image"
+                },
+                "pyramid_visualization": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Whether pyramid visualization should be returned.",
+                    "fa_icon": "far fa-image"
+                }
+            },
+            "fa_icon": "fas fa-images"
+        },
         "workflow_configuration": {
             "title": "Workflow configuration",
             "type": "object",
@@ -373,6 +417,12 @@
         {
             "$ref": "#/definitions/remote_sensing_image_options"
         },
+        {
+            "$ref": "#/definitions/higher_level_processing_modification"
+        },
+        {
+            "$ref": "#/definitions/visualization"
+        },
         {
             "$ref": "#/definitions/workflow_configuration"
         },

subworkflows/local/higher_level.nf

@@ -31,14 +31,21 @@ workflow HIGHER_LEVEL {
         trend_files_mosaic = trend_files.groupTuple()
 
         // visualizations
-        FORCE_MOSAIC( trend_files_mosaic, cube_file )
-        ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first())
-
-        FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif')  }.map { [ it[1].simpleName.substring(0,11), it[1] ] } .groupTuple() )
-        ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first())
+        mosaic_files = Channel.empty()
+        if (params.mosaic_visualization) {
+            FORCE_MOSAIC( trend_files_mosaic, cube_file )
+            mosaic_files = FORCE_MOSAIC.out.trend_files
+            ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first())
+        }
+
+        if (params.pyramid_visualization) {
+            FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif')  }.map { [ it[1].simpleName.substring(0,11), it[1] ] } .groupTuple() )
+            ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first())
+        }
 
     emit:
-        trend_files = FORCE_MOSAIC.out.trend_files
-        versions    = ch_versions
+        mosaic_files
+
+        versions = ch_versions
 
 }

workflows/rangeland.nf

@@ -128,7 +128,7 @@ workflow RANGELAND {
     HIGHER_LEVEL( PREPROCESSING.out.tiles_and_masks, cube_file, endmember_file )
     ch_versions = ch_versions.mix(HIGHER_LEVEL.out.versions)
 
-    grouped_trend_data = HIGHER_LEVEL.out.trend_files.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true )
+    grouped_trend_data = HIGHER_LEVEL.out.mosaic_files.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true )
 
     //
     // MODULE: Check results