a) Added some tSNE map options (#668) and b) switched from png to webp (

#667) for tSNE image format

www/api/resources/tsne_data.py

@@ -6,9 +6,9 @@
 import sys
 from math import ceil
 from pathlib import Path
-from time import sleep
 
 import geardb
+import matplotlib as mpl
 import matplotlib.colors as mcolors
 import matplotlib.pyplot as plt
 import numpy as np
@@ -57,6 +57,15 @@ def calculate_num_legend_cols(group_len):
     """Determine number of columns legend should have in tSNE plot."""
     return ceil(group_len / NUM_LEGENDS_PER_COL)
 
+def create_bluered_colorscale():
+    """Create the continuous plotly bluered colorscale."""
+    bluered = mcolors.LinearSegmentedColormap.from_list("bluered", [(0, "blue"), (1, "red")])
+    bluered_r = bluered.reversed()
+
+    # register the color map with matplotlib
+    register_colormap("bluered", bluered)
+    register_colormap("bluered_r", bluered_r)
+
 def create_colorscale_with_zero_gray(colorscale):
     """Take a predefined colorscale, and change the 0-value color to gray, and return."""
     from matplotlib import cm
@@ -102,13 +111,27 @@ def create_projection_adata(dataset_adata, dataset_id, projection_id):
 
 def create_projection_pca_colorscale():
     """Create a diverging colorscale but with black in the middle range."""
-    from matplotlib.colors import LinearSegmentedColormap
 
     # Src: https://matplotlib.org/stable/tutorials/colors/colormap-manipulation.html#directly-creating-a-segmented-colormap-from-a-list
     nodes = [0.0, 0.25, 0.4, 0.5, 0.6, 0.75, 1.0]
     colors = ["lightblue", "blue", "darkblue", "black", "darkred", "red", "lightcoral"]
-    return LinearSegmentedColormap.from_list("projection_pca", list(zip(nodes, colors)))
+    return mcolors.LinearSegmentedColormap.from_list("projection_pca", list(zip(nodes, colors)))
+
+def create_two_way_sorting(adata, gene_symbol):
+    """
+    Sorts the data in `adata` based on the absolute difference between the median value of `gene_symbol` and the values in `adata`.
 
+    Parameters:
+        adata (AnnData): Annotated data matrix.
+        gene_symbol (str): Symbol of the gene to sort by.
+
+    Returns:
+        AnnData: Sorted data matrix.
+    """
+    median = np.median(adata[:, gene_symbol].X.squeeze())
+    sort_order = np.argsort(np.abs(median - adata[:, gene_symbol].X.squeeze()))
+    ordered_obs = adata.obs.iloc[sort_order].index
+    return adata[ordered_obs, :]
 
 def get_colorblind_scale(n_colors):
     """Get a colorblind friendly colorscale (Viridis). Return n colors spaced equidistantly."""
@@ -174,6 +197,14 @@ def sort_legend(figure, sort_order, horizontal_legend=False):
 
     return (new_handles, new_labels)
 
+def register_colormap(name, cmap):
+    """Handle changes to matplotlib colormap interface in 3.6."""
+    try:
+        if name not in mpl.colormaps:
+            mpl.colormaps.register(cmap, name=name)
+    except AttributeError:
+        mpl.cm.register_cmap(name, cmap)
+
 # Rename axes labels to be whatever x and y fields were passed in
 # If axis labels are from an analysis, just use default embedding labels
 def rename_axes_labels(ax, x_axis, y_axis):
@@ -198,6 +229,9 @@ def post(self, dataset_id):
         skip_gene_plot = req.get('skip_gene_plot', False)
         plot_by_group = req.get('plot_by_group', None) # One expression plot per group
         max_columns = req.get('max_columns')   # Max number of columns before plotting to a new row
+        expression_palette = req.get('expression_palette', "YlOrRd")
+        reverse_palette = req.get('reverse_palette', False)
+        two_way_palette = req.get('two_way_palette', False) # If true, data extremes are in the forefront
         colors = req.get('colors')
         order = req.get('order', {})
         x_axis = req.get('x_axis', 'tSNE_1')   # Add here in case old tSNE plotly configs are missing axes data
@@ -206,6 +240,7 @@ def post(self, dataset_id):
         flip_y = req.get('flip_y', False)
         horizontal_legend = req.get('horizontal_legend', False)
         marker_size = req.get("marker_size", None)
+        center_around_median = req.get("center_around_median", False)
         filters = req.get('obs_filters', {})    # dict of lists
         session_id = request.cookies.get('gear_session_id')
         user = geardb.get_user_from_session_id(session_id)
@@ -367,6 +402,9 @@ def post(self, dataset_id):
                 os.remove(scanpy_copy)
             selected = selected[:, selected.var.index.duplicated() == False].copy(filename=scanpy_copy)
 
+        # Set the saved figure dpi based on the number of observations in the dataset after filtering
+        sc.settings.set_figure_params(dpi_save=int(150 + (selected.shape[0] / 500)))
+
         io_fig = None
         try:
             basis = PLOT_TYPE_TO_BASIS[plot_type]
@@ -381,21 +419,33 @@ def post(self, dataset_id):
             marker_size = int(marker_size)
 
         # Reverse cividis so "light" is at 0 and 'dark' is at incresing expression
-        expression_color = create_colorscale_with_zero_gray("cividis_r" if colorblind_mode else "YlOrRd")
+        if reverse_palette:
+            expression_palette += "_r"
 
-        # In projections, reorder so that the strongest weights (positive or negative) appear in forefront of plot
-        plot_sort_order = True
+        plot_sort_order = True   # scanpy auto-sorts by highest value by default
         plot_vcenter = None
+
+        if center_around_median:
+            plot_vcenter = np.median(selected[:, selected_gene].X.squeeze())
+
+        if two_way_palette:
+            selected = create_two_way_sorting(selected, selected_gene)
+            plot_sort_order = False
+
+        if expression_palette.startswith("bluered"):
+            create_bluered_colorscale()
+
+        expression_color = create_colorscale_with_zero_gray("cividis_r" if colorblind_mode else expression_palette)
+
+        # In projections, PCA is more meaningful with two-way sorting
         if projection_id:
             try:
                 algo = get_projection_algorithm(dataset_id, projection_id)
                 if algo == "pca":
-                    median = np.median(selected[:, selected_gene].X.squeeze())
-                    sort_order = np.argsort(np.abs(median - selected[:, selected_gene].X.squeeze()))
-                    ordered_obs = selected.obs.iloc[sort_order].index
-                    selected = selected[ordered_obs, :]
-                    plot_sort_order = False # scanpy auto-sorts by highest value by default so we need to override that
-                    plot_vcenter = median
+                    selected = create_two_way_sorting(selected, selected_gene)
+                    plot_sort_order = False
+                    # Since this is a diverging scale we want the center to be the median
+                    plot_vcenter = np.median(selected[:, selected_gene].X.squeeze())
                     expression_color = "cividis_r" if colorblind_mode else create_projection_pca_colorscale()
             except Exception as e:
                 print(str(e), file=sys.stderr)
@@ -554,7 +604,7 @@ def post(self, dataset_id):
 
         io_pic = io.BytesIO()
         io_fig.tight_layout()   # This crops out much of the whitespace around the plot. The next line does this with the legend too
-        io_fig.savefig(io_pic, format='png', bbox_inches="tight")
+        io_fig.savefig(io_pic, format='webp', bbox_inches="tight")
         io_pic.seek(0)
         plt.close() # Prevent zombie plots, which can cause issues
 

www/include/plot_config/post_plot/advanced_heatmap.html

@@ -1,4 +1,4 @@
-<section id="clustering_section_post" class="js-plot-config-section py-3">
+<section id="clustering_section_post" class="js-plot-config-section py-3 mb-1">
     <span class="subtitle is-4 is-underlined">Clustering</span>
     <p class="is-flex is-justify-content-space-between pr-3">
         <span class="has-text-weight-medium">Distance metric</span>
@@ -13,14 +13,14 @@
     </p>
     <p class="help">Select the metric to compute the pairwise distance for genes, and if applicable, observations.</p>
 
-    <label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
         <span class="has-text-weight-medium">Cluster observation samples</span>
         <input class="js-dash-cluster-obs" id="cluster_obs_post" type="checkbox">
     </label>
     <div class="alert alert-warning is-hidden" id="cluster_obs_warning">This dataset has more than 1000 observations, which may lead to the clustering step failing.  Please consider filtering your dataset categories beforehand.</div>
     <p class="help">NOTE: Disables sorting by primary and secondary categories.</p>
 
-    <label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
         <span class="has-text-weight-medium">Cluster genes</span>
         <input class="js-dash-cluster-genes" id="cluster_genes_post" type="checkbox">
     </label>
@@ -42,13 +42,13 @@
         <input class="js-dash-matrixplot" id="matrixplot_post" type="checkbox" checked>
     </label>
 
-    <label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
         <span class="has-text-weight-medium">Center mean expression around zero</span>
         <input class="js-dash-center-mean" id="center_around_zero_post" type="checkbox">
     </label>
     <p class="help">Choosing a diverging colorscale is recommended when enabling this option, and a sequential colorscale when disabling this option.</p>
 
-    <label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
         <span class="has-text-weight-medium">Flip genes and observation axes</span>
         <input class="js-dash-flip-axes" id="flip_axes_post" type="checkbox">
     </label>
@@ -69,7 +69,7 @@
         <input class="js-dash-enable-subsampling" id="enable_subsampling_post" type="checkbox">
     </label>
 
-    <p class="is-flex is-justify-content-space-between pr-3">
+    <p class="is-flex is-justify-content-space-between pr-3 mb-1">
         <span class="has-text-weight-medium">Subsampling limit</span>
         <input class="js-dash-subsampling-limit" id="subsampling_limit_post" type="number" value="0" min="0" disabled/>
     </p>

www/include/plot_config/post_plot/advanced_quadrant.html

@@ -18,25 +18,25 @@
         <select class="js-plot-req js-compare-groups js-dash-compare2" id="compare2_group_post"></select>
     </span>
 </p>
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Reference</span>
     <span class="select is-small">
         <select class="js-plot-req js-compare-groups js-dash-reference" id="reference_group_post"></select>
     </span>
 </p>
 <p class="help">All conditions must come from the same category.</p>
 
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
   <span class="has-text-weight-medium">Fold-change cutoff</span>
   <input class="js-dash-fc-cutoff" id="fc_cutoff_post" type="number" value="2" min="0" />
 </p>
 <p class="help">Choose the fold-change threshold that a query/reference comparison must exceed to be included. This value will be log2-transformed in plot.</p>
-<p class="is-flex is-justify-content-space-between pr-3 checkbox">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1c heckbox">
   <span class="has-text-weight-medium">FDR cutoff</span>
   <input class="js-dash-fdr-cutoff" id="fdr_cutoff_post" type="number" value="0.05" min="0" max="1" />
 </p>
 <p class="help">Choose the FDR (or qval) threshold a query/reference comparison must be under to be considered. Used in tandem with FDR cutoff.</p>
-<label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+<label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
   <span class="has-text-weight-medium">Include genes with FC of 0</span>
   <input class="js-dash-include-zero-fc" id="include_zero_fc_post" type="checkbox" checked>
 </label>

www/include/plot_config/post_plot/advanced_volcano.html

@@ -24,13 +24,13 @@
     <input class="js-dash-annot-nonsig" id="annot_nonsig_post" type="checkbox" checked>
 </label>
 
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Adjust p-value threshold</span>
     <input class="js-dash-pvalue-threshold" id="pvalue_threshold_post" type="number" value="0.05" min="0" step="0.01" />
 </p>
 <p class="help">Choose a p-value where all data points above the line are considered significant. This value will be -log10-transformed in the plot.</p>
 
-<label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+<label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
     <span class="has-text-weight-medium">Use adjusted p-values</span>
     <input class="js-dash-use-adj-pvalues" id="use_adj_pvalues_post" type="checkbox" checked>
 </label>
@@ -39,7 +39,7 @@
     <span class="has-text-weight-medium">Log2FC lower bounds</span>
     <input class="js-dash-lower-logfc-threshold" id="lower_logfc_threshold_post" type="number" value="-1" max="0" />
 </p>
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Log2FC upper bounds</span>
     <input class="js-dash-upper-logfc-threshold" id="upper_logfc_threshold_post" type="number" value="1" max="0" />
 </p>

www/include/plot_config/post_plot/multi_gene_as_data.html

@@ -1,7 +1,7 @@
 <!-- Include plot-specific HTML here -->
 <section class="js-plot-specific-options" id="post_plot_specific_options"></section>
 
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Differential expression test</span>
     <span class="select is-small">
         <select class="js-dash-de-test" id="de_test_post">

www/include/plot_config/post_plot/single_gene_plotly.html

@@ -51,7 +51,7 @@
         <span class="has-text-weight-medium">Axis min boundary</span>
         <input class="js-x-range js-plotly-x-min" id="xmin_value_post" type="number" />
     </p>
-    <p class="is-flex is-justify-content-space-between pr-3">
+    <p class="is-flex is-justify-content-space-between pr-3 mb-1">
         <span class="has-text-weight-medium">Axis max boundary</span>
         <input class="js-x-range js-plotly-x-max" id="xmax_value_post" type="number" />
     </p>
@@ -78,7 +78,7 @@
         <span class="has-text-weight-medium">Axis min boundary</span>
         <input class="js-y-range js-plotly-y-min" id="ymin_value_post" type="number" />
     </p>
-    <p class="is-flex is-justify-content-space-between pr-3">
+    <p class="is-flex is-justify-content-space-between pr-3 mb-1">
         <span class="has-text-weight-medium">Axis max boundary</span>
         <input class="js-y-range js-plotly-y-max" id="ymax_value_post" type="number" />
     </p>
@@ -126,7 +126,7 @@
             <select class="js-plotly-size" id="size_series_post"></select>
         </span>
     </p>
-    <p class="is-flex is-justify-content-space-between pr-3">
+    <p class="is-flex is-justify-content-space-between pr-3 mb-1">
         <span class="has-text-weight-medium">Min marker size</span>
         <input class="js-plotly-marker-size" id="marker_size_post" type="range" min="1" max="15" />
     </p>

www/include/plot_config/post_plot/tsne_static.html

@@ -27,15 +27,15 @@
     </span>
 </p>
 
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Plot per category group</span>
     <span class="select is-small">
         <select class="js-tsne-plot-by-series" id="plot_by_group_series_post" disabled></select>
     </span>
 </p>
 <p class="help">Split samples into category groups, then create plots per group.</p>
 
-<p class="is-flex is-justify-content-space-between pr-3">
+<p class="is-flex is-justify-content-space-between pr-3 mb-1">
     <span class="has-text-weight-medium">Number of subplot columns</span>
     <input class="js-tsne-max-columns" id="max_columns_post" type="number" min="0" max="9" value="3" disabled/>
 </p>
@@ -51,7 +51,7 @@
     <input class="js-tsne-horizontal-legend" id="horizontal_legend_post" type="checkbox" disabled>
 </label>
 
-<label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+<label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
     <span class="has-text-weight-medium">Override marker size auto-scaling</span>
     <input class="js-tsne-override-marker-size" id="override_marker_size_post" type="checkbox">
 </label>
@@ -60,3 +60,32 @@
     <span class="has-text-weight-medium">Choose marker size</span>
     <input class="js-tsne-marker-size" id="marker_size_post" type="range" min="1" max="100" value="25" disabled/>
 </p>
+
+<p class="is-flex is-justify-content-space-between pr-3">
+    <span class="has-text-weight-medium">Expression color palette</span>
+    <span class="select is-small">
+        <select class="js-tsne-color-palette" id="color_palette_post"></select>
+    </span>
+</p>
+<!-- Until I get it working
+<div class="field">
+    <label class="label has-text-weight-medium">Expression color palette</label>
+    <div class="control">
+        <div class="select is-fullwidth">
+            <select class="js-tsne-color-palette" id="color_palette_post" disabled></select>
+        </div>
+    </div>
+</div> -->
+<label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <span class="has-text-weight-medium">Reverse palette</span>
+    <input class="js-tsne-reverse-palette" id="reverse_palette_post" type="checkbox">
+</label>
+<label class="content is-flex is-justify-content-space-between pr-3 checkbox">
+    <span class="has-text-weight-medium">Check to bring data extremes to foreground</span>
+    <input class="js-tsne-two-way-palette" id="two_way_palette_post" type="checkbox">
+</label>
+<label class="content is-flex is-justify-content-space-between pr-3 mb-1 checkbox">
+    <span class="has-text-weight-medium">Center colorscale around median expression value</span>
+    <input class="js-tsne-center-around-median" id="center_around_median_post" type="checkbox">
+</label>
+<p class="help">Good when combined with a diverging colorscale.</p>

www/js/classes/tilegrid.js

@@ -1225,7 +1225,7 @@ class DatasetTile {
         plotContainer.append(tsnePreview);
 
         if (image) {
-            document.getElementById(tsnePreview.id ).setAttribute("src", `data:image/png;base64,${image}`);
+            document.getElementById(tsnePreview.id ).setAttribute("src", `data:image/webp;base64,${image}`);
         } else {
             console.warn(`Could not retrieve plot image for dataset display ${display.id}. Cannot make plot.`);
             return;