Merge branch 'develop'

KudryashevLab · Feb 16, 2023 · 6d354ac · 6d354ac
2 parents faaa9e4 + e2f5bf4
commit 6d354ac
Show file tree

Hide file tree

Showing 23 changed files with 3,837 additions and 354 deletions.
diff --git a/.github/workflows/wiki-mirror.yml b/.github/workflows/wiki-mirror.yml
@@ -0,0 +1,33 @@
+name: "Mirror wiki/ to repo.wiki"
+
+# Run on changes to the `wiki` directory on the `main` branch
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'wiki/**'
+
+jobs:
+  mirror:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v3
+        with:
+          path: TomoBEAR
+      - name: Checkout wiki
+        uses: actions/checkout@v3
+        with:
+          repository: "KudryashevLab/TomoBEAR.wiki"
+          path: TomoBEAR.wiki
+      - name: Copy wiki files
+        run: |
+          git config --global user.email "actions@github.com"
+          git config --global user.name "Wiki mirror"
+          cp $GITHUB_WORKSPACE/TomoBEAR/wiki/*.md $GITHUB_WORKSPACE/TomoBEAR.wiki
+          cd $GITHUB_WORKSPACE/TomoBEAR.wiki
+          git add .
+          # only commit if there are changes
+          git diff-index --quiet HEAD -- || git commit -m "sync from wiki/"
+          git push
diff --git a/.gitignore b/.gitignore
@@ -199,6 +199,7 @@ dynamo/matlab/src/+dpkproject/assemble_checkPeak.m
 dynamo/matlab/src/+dpkproject/+pipeline/+tables/gatherGPUforIteRef.m
 dynamo/matlab/src/dynamo_zaux_rescalings.m
 dynamo/matlab/src/+dpkproject/gatherTablesAfterProcess.m
+dynamo/matlab/src/dynamo_cc.m
 dynamo/matlab/src/dynamo_ccmatrix_assemble.m
 dynamo/matlab/src/dynamo_ccmatrix_compute.m
 dynamo/matlab/src/dynamo_iteration_check.m

diff --git a/README.md b/README.md
@@ -8,7 +8,8 @@ TomoBEAR is an automated, configurable and customizable full processing pipeline
 - (current one) [In situ Structural Biology Group](https://www.mdc-berlin.de/kudryashev) at the [MDCMM (Max Delbrück Center of Molecular Medicine)](https://www.mdc-berlin.de) in Berlin, Germany.
 
 ## Documentation
-Information on the installation, setup and usage can be found on [TomoBEAR wiki page](https://github.com/KudryashevLab/TomoBEAR/wiki).
+
+Information on the installation, setup and usage as well as tutorials and example results can be found in the corresponding [wiki](https://github.com/KudryashevLab/TomoBEAR/wiki).
 
 ## Repository structure
 User branches:
@@ -20,7 +21,8 @@ All the other branches are not intended for user usage! They hold non-stable dev
 In the future we are planning to introduce wiki versioning as well, stay tuned!
 
 ## Pipeline structure
-In the following picture you can see a flow chart which visualizes which steps `TomoBEAR` will and can do for you in an automated and parallel manner. `TomoBEAR` supports single nodes with GPUs and also some high-performance copmuter clusters.
+
+In the following picture you can see a flow chart which visualizes which steps `TomoBEAR` will and can do for you in an automated and parallel manner. `TomoBEAR` supports workstations and single interactive nodes with GPUs on the comuting clusters at the moment. We are also working towards enabling the support of computer clusters through a queue manager like SLURM or SGE (Sun Grid Engine).
 
 Note that it is not needed to start from raw data but you can also use already assembled tilt stacks if you provide the angular information.
 
@@ -33,23 +35,6 @@ Orange processing steps in the flow chart are mandatory and must be executed by
 
 In case of any questions or errors do not hesitate to contact one of the provided persons mentioned in the `Contacts` section below. Alternatively, for bug reports or feature suggestions you may use our [Issue Tracker](https://github.com/KudryashevLab/TomoBEAR/issues). Please, be polite and precise!
 
-## Results example (the tutorial case)
-
-The result shown here have been achieved in an automated manner. The only manual task which needed to be done is the choice of the classes for further processing in between the transition of different binning levels.
-
-### EMPIAR-10064
-
-On the EMPIAR-10064 dataset `TomoBEAR` achieved 11.25 Angstrom (with ~4k particles) as can be seen below on the FSC curve plot:
-<p align="center">
-<img src="https://raw.githubusercontent.com/KudryashevLab/TomoBEAR/main/images/ribosome_empiar_10064_fsc.jpg" alt="Ribosome EMPIAR-10064 FSC"/>
-</p>
-
-As well, below is provided ribosome final map view:
-
-<p align="center">
-<img src="https://raw.githubusercontent.com/KudryashevLab/TomoBEAR/main/images/ribosome_empiar_10064_map.png" alt="Ribosome EMPIAR-10064 Map"/>
-</p>
-
 # Acknowledgements
 
 We are grateful to the following organizations:

diff --git a/configurations/defaults.json b/configurations/defaults.json
@@ -218,25 +218,6 @@
         "reference_image_in_degreese": 0,
         "reconstruct": false
     },
-    "EMAN2Tomogram":{
-        "execution_method": "parallel",
-        "tltstep": 2.0,
-        "npk": 20,
-        "tltkeep": 0.9,
-        "outsize": "1k",
-        "niter": "2,1,1,1",
-        "bytile": true,
-        "pkkeep": 0.9,
-        "clipz": 500,
-        "bxsz": 32,
-        "correctrot": true,
-        "filterto": 0.45,
-        "rmbeadthr": -1.0,
-        "threads": 12,
-        "maxshift": 0.35,
-        "skip": false,
-        "citation": ""
-    },
     "Reconstruct": {
         "execution_method": "in_order",
         "reconstruct": "binned",
@@ -339,7 +320,8 @@
         "run_ctf_phase_flip": false,
         "reconstruct_tomograms": false,
         "defocus_tolerance": 20,
-        "iWidth": 2,
+        "iWidth": 15,
+        "maximum_strip_width": 1000,
         "ampContrast": 0.1,
         "defocus_file_version": 3,
         "defocus_file_version_3_flag": 1,
@@ -351,12 +333,6 @@
         "do_EPA": true,
         "estimated_astigmatism": 1000
     },
-    "ViewStacks": {
-        "execution_method": "sequential",
-        "show_truncated_stacks": true,
-        "skip": false,
-        "citation": ""
-    },
     "RefineFiducialModel": {
         "execution_method": "sequential",
         "skip": false,
@@ -599,11 +575,6 @@
         "skip": false,
         "citation": ""
     },
-    "BinarizeStacks": {
-        "execution_method": "parallel",
-        "skip": false,
-        "citation": ""
-    },
     "DynamoTiltSeriesAlignment": {
         "execution_method": "parallel",
         "use_newstack_for_binning": false,
@@ -722,52 +693,6 @@
         "citation_link": "",
         "doi": ""
     },
-    "CalculateBinarizationThreshold": {
-        "execution_method": "once",
-        "skip": false,
-        "citation": ""
-    },
-    "SUSANCTFCorrection": {
-        "execution_method": "once",
-        "ctf_estimation_box_size": 512,
-        "ctf_estimation_def_min": 20000,
-        "ctf_estimation_def_max": 50000
-    },
-    "SUSAN": {
-        "index": 0,
-        "execution_method": "once",
-        "do_ctf_correction": false,
-        "ctf_estimation_box_size": 512,
-        "ctf_estimation_def_min": 20000,
-        "ctf_estimation_def_max": 50000,
-        "extra_padding": 12,
-        "rec_halves": false,
-        "ssnr_s": 1,
-        "reconstruction_box_size": 512,
-        "bandpass_cut_off_resolution_in_angstrom": 20,
-        "bandpass_cut_on_fourier_pixel": 2,
-        "iterations": 10,
-        "alignment_type": 3,
-        "cc_threshold": 0.9,
-        "ellipsoid_size": [10, 10, 10],
-        "classes": 0,
-        "class_distribution": [0.25, 0.25, 0.25, 0.25],
-        "use_symmetry": false,
-        "weight_projections": true,
-        "checkpoint_module": false,
-        "last_classification_binning": 4,
-        "discretization_bias": 1.5,
-        "noise_scaling_factor": 0.2,
-        "use_mask_from_template_matching": false,
-        "mask_radii_ratio": [1, 1, 0.5],
-        "classification_convergence_iterations": 3,
-        "allow_drift": true,
-        "determenistic_class_assignment": true,
-        "refine_factor": 2,
-        "refine_level": 0,
-        "refine_factor_susan": 1,
-        "skip_if_no_class_changes_since_iterations": 3
-    },
     "DynamoAlignmentProject": {
         "randomize_angles": false,
         "bf": 4,
@@ -910,33 +835,6 @@
         "show_figures": false,
         "save_figures": true
     },
-    "SPOC": {
-        "execution_method": "once",
-        "repository_path": "",
-        "spoc_env": "spoc",
-        "numAsymUnits": 1,
-        "mask_path": ""
-    },
-    "GenerateParticles": {
-        "execution_method": "once",
-        "use_SUSAN": true,
-        "ctf_correction_method": "SUSAN",
-        "SUSAN_ctf_box_size": 400,
-        "SUSAN_binning": 0,
-        "SUSAN_defocus_min": 10000,
-        "SUSAN_defocus_max": 50000,
-        "binning": 0,
-        "susan_lowpass": 65,
-        "susan_padding": 200,
-        "per_particle_ctf_correction": "phase_flip",
-        "padding_policy": "noise",
-        "normalization": "zm",
-        "use_symmetry": true,
-        "average_particles": false,
-        "table": "",
-        "table": "",
-        "all_in_one_folder": true
-    },
     "CryoCARE": {
         "execution_method": "once",
         "repository_path": "",
@@ -955,43 +853,6 @@
         "patch_shape": 512,
         "tomograms_to_train_on": 0
     },
-    "EstimateTomogramResolution": {
-        "execution_method": "once",
-        "binning": 0,
-        "use_SUSAN": true,
-        "ctf_correction_method": "SUSAN",
-        "SUSAN_ctf_box_size": 400,
-        "SUSAN_binning": 0,
-        "SUSAN_defocus_min": 10000,
-        "SUSAN_defocus_max": 50000,
-        "susan_padding": 200,
-        "per_particle_ctf_correction": "phase_flip",
-        "padding_policy": "noise",
-        "normalization": "zm",
-        "save_figures": true,
-        "show_figures": false,
-        "box_size": 150,
-        "overlap": 1,
-        "offset": 0
-    },
-    "GumNet": {
-        "execution_method": "once",
-        "repository_path": "",
-        "gum_net_env": "",
-        "binning": 0,
-        "use_SUSAN": true,
-        "ctf_correction_method": "SUSAN",
-        "SUSAN_ctf_box_size": 400,
-        "SUSAN_binning": 0,
-        "SUSAN_defocus_min": 10000,
-        "SUSAN_defocus_max": 50000,
-        "susan_padding": 0,
-        "per_particle_ctf_correction": "phase_flip",
-        "padding_policy": "noise",
-        "normalization": "zm",
-        "use_delta_shifts": false,
-        "box_size": 0
-    },
     "DeepFinder": {
         "execution_method": "once",
         "checkpoint_module": true,
@@ -1019,6 +880,8 @@
         "reconstruction": false,
         "weighted_back_projection": true,
         "tilt_axis_refine_flag": 1,
+        "apply_tilt_axis_offset": 0,
+        "tilt_axis_offset": 0,
         "align_height_ratio": 0.75,
         "apply_dose_weighting": false,
         "sart": "20 5",
@@ -1033,5 +896,19 @@
         "execution_method": "once",
         "checkpoint_module": true,
         "binning": 0
+    },
+    "GridEdgeEraser": {
+        "execution_method": "in_order",
+        "detection_binning": 4,
+        "grid_hole_diameter_in_um": 2,
+        "output_shift_user": [0, 0],
+        "output_shift_kernel_factor": [0, 0],
+        "binarize_threshold_in_std": 3,
+        "grid_detection_threshold_in_std": 3,
+        "smooth_mask_border": true,
+        "smooth_to_mean": true,
+        "smoothing_exp_decay": -40,
+        "cleaned_postfix": "gef",
+        "relink_as_previous_output": false
     }
 }
diff --git a/configurations/ribosome_empiar_10064_dynamo.json b/configurations/ribosome_empiar_10064_dynamo.json
@@ -7,6 +7,7 @@
         "expected_symmetrie": "C1",
         "apix": 2.62,
         "tilt_angles": [-60.0, -58.0, -56.0, -54.0, -52.0, -50.0, -48.0, -46.0, -44.0, -42.0, -40.0, -38.0, -36.0, -34.0, -32.0, -30.0, -28.0, -26.0, -24.0, -22.0, -20.0, -18.0, -16.0, -14.0, -12.0, -10.0, -8.0, -6.0, -4.0, -2.0, 0.0, 2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0, 18.0, 20.0, 22.0, 24.0, 26.0, 28.0, 30.0, 32.0, 34.0, 36.0, 38.0, 40.0, 42.0, 44.0, 46.0, 48.0, 50.0, 52.0, 54.0, 56.0, 58.0, 60.0],
+        "rotation_tilt_axis": -5,
         "gold_bead_size_in_nm": 9,
         "template_matching_binning": 8,
         "binnings": [2, 4, 8],