Merge pull request #2184 from broadinstitute/development

Release 1.87.0

Gemfile.lock

@@ -616,4 +616,4 @@ RUBY VERSION
    ruby 3.2.2p53
 
 BUNDLED WITH
-   2.3.19
+   2.6.1

app/controllers/api/v1/visualization/explore_controller.rb

@@ -158,6 +158,7 @@ def self.get_study_explore_info(study, user)
             differentialExpression: AnnotationVizService.differential_expression_menu_opts(study),
             hasImageCache: study.cluster_groups.where(has_image_cache: true).pluck(:name),
             clusterPointAlpha: study.default_cluster_point_alpha,
+            expressionSort: study.default_expression_sort,
             colorProfile: study.default_color_profile,
             bucketId: study.bucket_id,
             isPublic: study.public

app/controllers/site_controller.rb

@@ -728,7 +728,7 @@ def study_params
                                   :default_options => [:cluster, :annotation, :color_profile, :expression_label,
                                                        :deliver_emails, :cluster_point_size, :cluster_point_alpha,
                                                        :cluster_point_border, :precomputed_heatmap_label,
-                                                       override_viz_limit_annotations: []],
+                                                       :expression_sort, override_viz_limit_annotations: []],
                                   study_shares_attributes: [:id, :_destroy, :email, :permission],
                                   study_detail_attributes: [:id, :full_description],
                                   reviewer_access_attributes: [:id, :expires_at],

app/controllers/studies_controller.rb

@@ -1041,7 +1041,8 @@ def directory_listing_params
   def default_options_params
     params.require(:study_default_options).permit(:cluster, :annotation, :color_profile, :expression_label,
                                                   :cluster_point_size, :cluster_point_alpha, :cluster_point_border,
-                                                  :precomputed_heatmap_label, override_viz_limit_annotations: [])
+                                                  :precomputed_heatmap_label, :expression_sort,
+                                                  override_viz_limit_annotations: [])
   end
 
   def set_file_types

app/javascript/components/explore/ExploreDisplayPanelManager.jsx

@@ -21,6 +21,7 @@ import DifferentialExpressionModal from '~/components/explore/DifferentialExpres
 import CellFilteringModal from '~/components/explore/CellFilteringModal'
 import { CellFilteringPanel, CellFilteringPanelHeader } from './CellFilteringPanel'
 import BookmarkManager from '~/components/bookmarks/BookmarkManager'
+import { EXPRESSION_SORT_OPTIONS } from '~/components/visualization/PlotDisplayControls'
 
 /** Get the selected clustering and annotation, or their defaults */
 function getSelectedClusterAndAnnot(exploreInfo, exploreParams) {
@@ -185,6 +186,11 @@ function getIsAuthorDe(exploreInfo, exploreParams) {
   return isAuthorDe
 }
 
+// get the current value for expression sorting (or study default, if present)
+function getExpressionSort(exploreInfo, exploreParams) {
+  return exploreParams?.expressionSort || exploreInfo?.expressionSort || EXPRESSION_SORT_OPTIONS[0]
+}
+
 /**
  * Manages the right panel section of the explore view of a study. We have three options for the right side
  * panel with different controls to adjust the plots: Options (or default), Differential Expression, and
@@ -226,6 +232,7 @@ export default function ExploreDisplayPanelManager({
   const hasPairwiseDe = getHasComparisonDe(exploreInfo, exploreParams, 'pairwise')
   const deHeaders = getDeHeaders(exploreInfo, exploreParams)
   const isAuthorDe = getIsAuthorDe(exploreInfo, exploreParams)
+  const expressionSort = getExpressionSort(exploreInfo, exploreParams)
 
   // exploreParams object without genes specified, to pass to cluster comparison plots
   const referencePlotDataParams = _clone(exploreParams)
@@ -514,6 +521,7 @@ export default function ExploreDisplayPanelManager({
               shownTab={shownTab}
               exploreParams={exploreParamsWithDefaults}
               updateExploreParams={updateExploreParams}
+              expressionSort={expressionSort}
               allGenes={exploreInfo ? exploreInfo.uniqueGenes : []}/>
             <div id={useBookmarkContainer ? 'bookmark-container' : ''} className={useRowClass ? 'row' : ''}>
               <div className={useRowClass ? 'col-xs-12' : ''}>

app/javascript/components/explore/ExploreTabRouter.jsx

@@ -117,7 +117,7 @@ function buildExploreParamsFromQuery(query) {
       exploreParams.expressionFilter = filterArray
     }
   }
-
+  exploreParams.expressionSort = queryParams.expressionSort
   exploreParams.hiddenTraces = queryParams.hiddenTraces ? queryParams.hiddenTraces.split(',') : []
   exploreParams.isSplitLabelArrays = queryParams.isSplitLabelArrays === 'true' ? true : null
 
@@ -145,6 +145,7 @@ function buildQueryFromParams(exploreParams) {
     trackFileName: exploreParams.trackFileName,
     ideogramFileId: exploreParams.ideogramFileId,
     expressionFilter: exploreParams.expressionFilter ? exploreParams.expressionFilter.join(FILTER_RANGE_DELIMITER) : undefined,
+    expressionSort: exploreParams.expressionSort,
     hiddenTraces: exploreParams.hiddenTraces.join(','),
     isSplitLabelArrays: exploreParams.isSplitLabelArrays ? 'true' : undefined,
     facets: exploreParams.facets
@@ -166,7 +167,7 @@ function buildQueryFromParams(exploreParams) {
 const PARAM_LIST_ORDER = [
   'geneList', 'genes', 'cluster', 'spatialGroups', 'annotation', 'subsample', 'consensus',
   'tab', 'scatterColor', 'distributionPlot', 'distributionPoints',
-  'heatmapFit', 'heatmapRowCentering', 'trackFileName', 'ideogramFileId', 'expressionFilter',
+  'heatmapFit', 'heatmapRowCentering', 'trackFileName', 'ideogramFileId', 'expressionFilter', 'expressionSort',
   'isSplitLabelArrays', 'hiddenTraces', 'facets'
 ]
 /** sort function for passing to stringify to ensure url params are specified in a user-friendly order */

app/javascript/components/explore/ScatterTab.jsx

@@ -25,6 +25,8 @@ export default function ScatterTab({
     updateExploreParamsWithDefaults({ scatterColor: color }, false)
   }
 
+  const expressionSort = exploreParamsWithDefaults?.expressionSort || exploreInfo?.expressionSort
+
   // identify any repeat graphs
   const newContextMap = getNewContextMap(scatterParams, plotlyContextMap.current)
 
@@ -55,6 +57,7 @@ export default function ScatterTab({
               canEdit={exploreInfo.canEdit}
               bucketId={exploreInfo.bucketId}
               filteredCells={filteredCells}
+              expressionSort={expressionSort}
               dimensionProps={{
                 isMultiRow,
                 isTwoColumn: isTwoColRow,

app/javascript/components/visualization/PlotDisplayControls.jsx

@@ -5,6 +5,9 @@ import { Slider, Rail, Handles, Tracks, Ticks } from 'react-compound-slider'
 import Select from '~/lib/InstrumentedSelect'
 import { Handle, Track, Tick } from '~/components/search/controls/slider/components'
 import PlotOptions from './plot-options'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
+import { faExternalLinkSquareAlt, faInfoCircle } from '@fortawesome/free-solid-svg-icons'
+import { Popover, OverlayTrigger } from 'react-bootstrap'
 const {
   SCATTER_COLOR_OPTIONS, defaultScatterColor, DISTRIBUTION_PLOT_OPTIONS, DISTRIBUTION_POINTS_OPTIONS,
   ROW_CENTERING_OPTIONS, FIT_OPTIONS
@@ -28,9 +31,9 @@ const railStyle = {
 // the code is in because it allows easier testing of the trace filtering logic implemented in plot.js
 const ENABLE_EXPRESSION_FILTER = false
 
-
+export const EXPRESSION_SORT_OPTIONS = ['high', 'low', 'unsorted']
 /** the graph customization controls for the exlore tab */
-export default function RenderControls({ shownTab, exploreParams, updateExploreParams, allGenes }) {
+export default function RenderControls({ shownTab, exploreParams, updateExploreParams, expressionSort, allGenes }) {
   const scatterColorValue = exploreParams.scatterColor ? exploreParams.scatterColor : defaultScatterColor
   let distributionPlotValue = DISTRIBUTION_PLOT_OPTIONS.find(opt => opt.value === exploreParams.distributionPlot)
   if (!distributionPlotValue) {
@@ -57,6 +60,16 @@ export default function RenderControls({ shownTab, exploreParams, updateExploreP
   const showColorScale = !!(showScatter && (exploreParams.annotation.type === 'numeric' || exploreParams.genes.length))
   const filterValues = exploreParams.expressionFilter ?? [0, 1]
   const showExpressionFilter = ENABLE_EXPRESSION_FILTER && exploreParams.genes.length && showScatter
+  const showExpressionSort = exploreParams.genes.length > 0 && showScatter
+
+  const expressionSortPopover = <Popover id={`expression-sort-popover`}>
+    Specify which cells to bring to the front of expression-based scatter plots based on expression value.&nbsp;
+    <a href="https://singlecell.zendesk.com/hc/en-us/articles/31772258040475" target="_blank">Learn more</a>.
+  </Popover>
+  const sortDocumentationLink =
+    <OverlayTrigger trigger={['hover', 'focus']} rootClose placement="left" overlay={expressionSortPopover} delayHide={1500}>
+        <a className="action help-icon"><FontAwesomeIcon icon={faInfoCircle}/></a>
+    </OverlayTrigger>
 
   return (
     <div>
@@ -65,13 +78,36 @@ export default function RenderControls({ shownTab, exploreParams, updateExploreP
           <span className="detail"> (for numeric data)</span>
           <Select
             data-analytics-name="scatter-color-picker"
-            options={SCATTER_COLOR_OPTIONS.map(opt => ({ label: opt, value: opt }))}
-            value={{ label: scatterColorValue, value: scatterColorValue }}
+            options={SCATTER_COLOR_OPTIONS.map(opt => ({
+              label: opt,
+              value: opt
+            }))}
+            value={{
+              label: scatterColorValue,
+              value: scatterColorValue
+            }}
             clearable={false}
             onChange={option => updateExploreParams({ scatterColor: option.value })}/>
         </label>
-      </div> }
-      { showExpressionFilter && <div className="render-controls">
+      </div>}
+      {showExpressionSort && <div className="render-controls">
+        <label className="labeled-select">Order expression by&nbsp;
+          {sortDocumentationLink}
+          <Select
+            data-analytics-name="expression-sort-select"
+            options={EXPRESSION_SORT_OPTIONS.map(opt => ({
+              label: opt,
+              value: opt
+            }))}
+            value={{
+              label: expressionSort,
+              value: expressionSort
+            }}
+            clearable={false}
+            onChange={option => updateExploreParams({ expressionSort: option.value })}/>
+        </label>
+      </div>}
+      {showExpressionFilter && <div className="render-controls">
         <label>Expression filter</label>
         <Slider
           mode={1}

app/javascript/components/visualization/ScatterPlot.jsx

@@ -46,7 +46,7 @@ window.Plotly = Plotly
 function RawScatterPlot({
   studyAccession, cluster, annotation, subsample, consensus, genes, scatterColor, dimensionProps,
   isAnnotatedScatter=false, isCorrelatedScatter=false, isCellSelecting=false, plotPointsSelected, dataCache,
-  canEdit, bucketId, expressionFilter=[0, 1], setCountsByLabelForDe, hiddenTraces=[],
+  canEdit, bucketId, expressionFilter=[0, 1], expressionSort, setCountsByLabelForDe, hiddenTraces=[],
   isSplitLabelArrays, updateExploreParams, filteredCells
 }) {
   const [countsByLabel, setCountsByLabel] = useState(null)
@@ -199,6 +199,7 @@ function RawScatterPlot({
       scatter,
       activeTraceLabel,
       expressionFilter,
+      expressionSort,
       isSplitLabelArrays: isSplitLabelArrays ?? scatter.isSplitLabelArrays,
       isRefGroup: isRG,
       originalLabels,
@@ -501,7 +502,7 @@ function RawScatterPlot({
     fetchData()
   }, [
     cluster, loadedAnnotation, subsample, genes.join(','), isAnnotatedScatter, consensus,
-    filteredCells?.join(',')
+    filteredCells?.join(','), expressionSort
   ])
 
   useUpdateEffect(() => {
@@ -721,6 +722,7 @@ function getPlotlyTraces({
   },
   activeTraceLabel,
   expressionFilter,
+  expressionSort,
   isSplitLabelArrays,
   isRefGroup,
   originalLabels,
@@ -777,7 +779,7 @@ function getPlotlyTraces({
     let workingTrace = traces[0]
     let colors
     if (isGeneExpressionForColor) {
-      workingTrace = sortTraceByExpression(workingTrace)
+      workingTrace = sortTraceByExpression(workingTrace, expressionSort)
       colors = workingTrace.expression
     } else {
       colors = isGeneExpressionForColor ? workingTrace.expression : workingTrace.annotations

app/javascript/lib/plot.js

@@ -245,8 +245,38 @@ PlotUtils.sortTraces = function(traces, activeTraceLabel) {
   return traces.sort(traceCountsSort)
 }
 
-/** sort the passsed in trace by expression value */
-PlotUtils.sortTraceByExpression = function(trace) {
+/**
+ * sorting algorithm that prepends zero values before applying sort logic
+ * this causes non-zero expression to be plotted on top of zeros in specified order
+ *
+ * @param a first element to compare
+ * @param b second element to compare
+ * @param sortOrder order of sorting preference (high, low)
+ * @returns {number}
+ */
+PlotUtils.weightedZeroSort = function(a, b, sortOrder) {
+  if (a === 0) {
+    return -1
+  } else if (b === 0) {
+    return 1
+  }
+
+  if (sortOrder === 'low') {
+    return b - a
+  } else {
+    return a - b
+  } 
+}
+
+/**
+ * sort the passed in trace by expression value
+ *
+ * @param trace {Object} Plotly trace data
+ * @param sortOrder {String} order of sorting preference (high, low, or unsorted)
+ *
+ * */
+PlotUtils.sortTraceByExpression = function(trace, sortOrder) {
+  if (sortOrder === 'unsorted') { return trace }
   const hasZ = !!trace.z
   const traceLength = trace.x.length
   const sortedTrace = {
@@ -258,7 +288,7 @@ PlotUtils.sortTraceByExpression = function(trace) {
   for (let i = 0; i < traceLength; i++) {
     expressionsWithIndices[i] = [trace.expression[i], i]
   }
-  expressionsWithIndices.sort((a, b) => a[0] - b[0])
+  expressionsWithIndices.sort(function(a,b) { return PlotUtils.weightedZeroSort(a[0], b[0], sortOrder) })
 
   // initialize the other arrays with their size
   // (see https://codeabitwiser.com/2015/01/high-performance-javascript-arrays-pt1/ for performance rationale)

app/javascript/styles/_brand.scss

@@ -150,6 +150,23 @@
   height: 54px;
   padding: 5px 30px 0 15px;
   display: inline-block;
+
+  a {
+    margin-right: 17px;
+  }
+}
+
+.bluesky-logo {
+  color: white;
+  position: relative;
+  top: 8px;
+  left: 3px;
+}
+
+.bluesky-link {
+  color: transparent !important;
+  background-color: transparent !important;
+  margin-top: 10px;
 }
 
 .study-panel, .study-genes-panel {

app/lib/cluster_cache_service.rb

@@ -40,6 +40,10 @@ def self.cache_all_defaults
   #   - (JSON) => ActionDispatch::Cache entry of JSON viz data
   def self.cache_study_defaults(study)
     Rails.logger.info "Checking defaults on #{study.accession} for pre-caching"
+    unless study.can_visualize_clusters?
+      Rails.logger.info "#{study.accession} cannot visualize clusters, skipping"
+      return nil
+    end
     begin
       cluster = study.default_cluster
       annotation = study.default_annotation