Deck migration/filter visible environment values

libs/deck-layers/src/layers/fourwings/FourwingsHeatmapLayer.ts

@@ -89,6 +89,8 @@ export class FourwingsHeatmapLayer extends CompositeLayer<FourwingsHeatmapLayerP
       aggregationOperation,
       tilesCache,
       scales,
+      minVisibleValue,
+      maxVisibleValue,
     } = this.props
     if (!data || !colorDomain || !colorRanges || !tilesCache) {
       return []
@@ -124,7 +126,11 @@ export class FourwingsHeatmapLayer extends CompositeLayer<FourwingsHeatmapLayerP
           chosenValueIndex = index
         }
       })
-      if (!chosenValue) {
+      if (
+        !chosenValue ||
+        (minVisibleValue && chosenValue < minVisibleValue) ||
+        (maxVisibleValue && chosenValue > maxVisibleValue)
+      ) {
         target = EMPTY_CELL_COLOR
         return target
       }
@@ -187,7 +193,15 @@ export class FourwingsHeatmapLayer extends CompositeLayer<FourwingsHeatmapLayerP
           getPolygonOffset: (params: any) => getLayerGroupOffset(LayerGroup.Heatmap, params),
           updateTriggers: {
             // This tells deck.gl to recalculate fillColor on changes
-            getFillColor: [startTime, endTime, colorDomain, colorRanges, comparisonMode],
+            getFillColor: [
+              startTime,
+              endTime,
+              colorDomain,
+              colorRanges,
+              comparisonMode,
+              minVisibleValue,
+              maxVisibleValue,
+            ],
           },
         })
       ),

libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts

@@ -22,21 +22,16 @@ import {
   FourwingsStaticFeature,
   FourwingsStaticFeatureProperties,
 } from '@globalfishingwatch/deck-loaders'
-import {
-  HEATMAP_COLOR_RAMPS,
-  rgbaStringToComponents,
-  ColorRampsIds,
-} from '@globalfishingwatch/layer-composer'
 import { filterFeaturesByBounds } from '@globalfishingwatch/data-transforms'
-import { COLOR_RAMP_DEFAULT_NUM_STEPS } from '../../utils/colorRamps'
+import { COLOR_RAMP_DEFAULT_NUM_STEPS, ColorRampId, getColorRamp } from '../../utils/colorRamps'
 import {
   COLOR_HIGHLIGHT_LINE,
   GFWMVTLoader,
   LayerGroup,
-  deckToRgbaColor,
   getLayerGroupOffset,
+  rgbaStringToComponents,
 } from '../../utils'
-import { EMPTY_CELL_COLOR, filterElementByPercentOfIndex } from './fourwings.utils'
+import { EMPTY_CELL_COLOR, filterCells } from './fourwings.utils'
 import {
   FOURWINGS_MAX_ZOOM,
   HEATMAP_API_TILES_URL,
@@ -46,7 +41,6 @@ import {
   FourwingsHeatmapTileLayerProps,
   FourwingsTileLayerState,
   FourwingsAggregationOperation,
-  FourwinsTileLayerScale,
   FourwingsHeatmapStaticLayerProps,
   FourwingsPickingInfo,
   FourwingsPickingObject,
@@ -66,14 +60,13 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
 > {
   static layerName = 'FourwingsHeatmapStaticLayer'
   static defaultProps = defaultProps
-  scale: typeof scaleLinear | undefined = undefined
 
   initializeState(context: LayerContext) {
     super.initializeState(context)
     this.state = {
       colorDomain: [],
       colorRanges: this._getColorRanges(),
-      scale: scaleLinear([], []),
+      scales: [],
     }
   }
 
@@ -83,7 +76,7 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
 
   _getColorRanges = () => {
     return this.props.sublayers.map(({ colorRamp }) =>
-      HEATMAP_COLOR_RAMPS[colorRamp as ColorRampsIds].map((c) => rgbaStringToComponents(c))
+      getColorRamp({ rampId: colorRamp as ColorRampId })
     )
   }
 
@@ -92,36 +85,27 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
     // NO_DATA_VALUE = 0
     // SCALE_VALUE = 0.01
     // OFFSET_VALUE = 0
+    const { minVisibleValue, maxVisibleValue } = this.props
     const currentZoomData = this.getData()
     if (!currentZoomData.length) {
       return this.getColorDomain()
     }
-    const dataSample =
-      currentZoomData.length > MAX_RAMP_VALUES_PER_TILE
-        ? currentZoomData.filter(filterElementByPercentOfIndex)
-        : currentZoomData
-
-    const allValues = dataSample.flatMap((feature) => {
-      if (
-        (this.props.minVisibleValue && feature.properties.count < this.props.minVisibleValue) ||
-        (this.props.maxVisibleValue && feature.properties.count > this.props.maxVisibleValue)
-      ) {
-        return []
-      }
-      return feature.properties?.count
-    })
+    const values = currentZoomData.flatMap((d) => d.properties?.count || [])
+    const allValues =
+      values.length > MAX_RAMP_VALUES_PER_TILE
+        ? values.filter((d, i) => filterCells(d, i, minVisibleValue, maxVisibleValue))
+        : values
 
     const steps = ckmeans(allValues, Math.min(allValues.length, COLOR_RAMP_DEFAULT_NUM_STEPS)).map(
       (step) => step[0]
     )
-
     return steps
   }
 
   _updateColorDomain = () => {
     const colorDomain = this._calculateColorDomain() as number[]
-    const colorRanges = this._getColorRanges()?.[0]?.map((c) => deckToRgbaColor(c))
-    this.setState({ colorDomain, scale: scaleLinear(colorDomain, colorRanges) })
+    const colorRanges = this._getColorRanges()[0]
+    this.setState({ colorDomain, scales: [scaleLinear(colorDomain, colorRanges)] })
   }
 
   debouncedUpdateColorDomain = debounce(() => {
@@ -152,23 +136,22 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
   }
 
   getFillColor = (feature: Feature<Geometry, FourwingsStaticFeatureProperties>) => {
-    if (!this.state.scale) {
-      return EMPTY_CELL_COLOR
-    }
-    if (!feature.properties.count) {
-      // TODO make the filter for the visible data here
-      return EMPTY_CELL_COLOR
-    }
+    const { scales } = this.state as FourwingsTileLayerState
+    const scale = scales?.[0]
     if (
+      !scale ||
+      !feature.properties.count ||
       (this.props.minVisibleValue && feature.properties.count < this.props.minVisibleValue) ||
       (this.props.maxVisibleValue && feature.properties.count > this.props.maxVisibleValue)
     ) {
       return EMPTY_CELL_COLOR
     }
-    const value = (this.state.scale as FourwinsTileLayerScale)(feature.properties.count)
+
+    const value = scale(feature.properties.count)
     if (!value) {
       return EMPTY_CELL_COLOR
     }
+
     return rgbaStringToComponents(value) as Color
   }
 
@@ -185,6 +168,7 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
     const { tilesUrl, sublayers, resolution, minVisibleValue, maxVisibleValue, maxZoom } =
       this.props
     const { colorDomain, colorRanges, scales } = this.state as FourwingsTileLayerState
+    const scale = scales?.[0]
     const params = {
       datasets: sublayers.flatMap((sublayer) => sublayer.datasets),
       format: 'MVT',
@@ -204,7 +188,7 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer<
         getPolygonOffset: (params) => getLayerGroupOffset(LayerGroup.HeatmapStatic, params),
         getFillColor: this.getFillColor,
         updateTriggers: {
-          getFillColor: [colorDomain, colorRanges, scales, minVisibleValue, maxVisibleValue],
+          getFillColor: [colorDomain, colorRanges, scale, minVisibleValue, maxVisibleValue],
         },
       }),
     ]

libs/deck-layers/src/layers/fourwings/FourwingsHeatmapTileLayer.ts

@@ -18,20 +18,20 @@ import {
   FourwingsLoader,
   ParseFourwingsOptions,
 } from '@globalfishingwatch/deck-loaders'
-import { HEATMAP_COLOR_RAMPS, ColorRampsIds } from '@globalfishingwatch/layer-composer'
 import { GFWAPI } from '@globalfishingwatch/api-client'
 import { filterFeaturesByBounds } from '@globalfishingwatch/data-transforms'
 import {
   COLOR_RAMP_BIVARIATE_NUM_STEPS,
   COLOR_RAMP_DEFAULT_NUM_STEPS,
   ColorRampId,
   getBivariateRamp,
+  getColorRamp,
 } from '../../utils/colorRamps'
 import {
   aggregateCellTimeseries,
-  filterElementByPercentOfIndex,
   getFourwingsChunk,
   getDataUrlBySublayer,
+  filterCells,
 } from './fourwings.utils'
 import { FourwingsHeatmapLayer } from './FourwingsHeatmapLayer'
 import {
@@ -88,11 +88,8 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer<
     if (this.props.comparisonMode === FourwingsComparisonMode.Bivariate) {
       return getBivariateRamp(this.props.sublayers.map((s) => s?.colorRamp) as ColorRampId[])
     }
-    return this.props.sublayers.map(
-      ({ colorRamp }) =>
-        HEATMAP_COLOR_RAMPS[
-          (this.props.colorRampWhiteEnd ? `${colorRamp}_toWhite` : colorRamp) as ColorRampsIds
-        ]
+    return this.props.sublayers.map(({ colorRamp }) =>
+      getColorRamp({ rampId: colorRamp as ColorRampId, whiteEnd: this.props.colorRampWhiteEnd })
     )
   }
 
@@ -101,34 +98,36 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer<
     // NO_DATA_VALUE = 0
     // SCALE_VALUE = 0.01
     // OFFSET_VALUE = 0
-    const { comparisonMode, aggregationOperation } = this.props
+    const { comparisonMode, aggregationOperation, minVisibleValue, maxVisibleValue } = this.props
     const currentZoomData = this.getData()
     if (!currentZoomData.length) {
       return this.getColorDomain()
     }
     const dataSample =
       currentZoomData.length > MAX_RAMP_VALUES_PER_TILE
-        ? currentZoomData.filter(filterElementByPercentOfIndex)
+        ? currentZoomData.filter((d, i) => filterCells(d, i))
         : currentZoomData
 
     // TODO:deck remove this and calculate values equal to Compare
     if (comparisonMode === FourwingsComparisonMode.Bivariate) {
       let allValues: [number[], number[]] = [[], []]
       dataSample.forEach((feature) => {
         feature.properties?.values.forEach((sublayerValues, sublayerIndex) => {
-          allValues[sublayerIndex].push(...sublayerValues.filter(filterElementByPercentOfIndex))
+          allValues[sublayerIndex].push(...sublayerValues.filter((d, i) => filterCells(d, i)))
         })
       })
       if (!allValues.length) {
         return this.getColorDomain()
       }
 
-      const steps = allValues.filter((sublayer) => sublayer.length).map((sublayerValues) =>
-        ckmeans(
-          sublayerValues,
-          Math.min(sublayerValues.length, COLOR_RAMP_BIVARIATE_NUM_STEPS)
-        ).map((step) => step[0])
-      )
+      const steps = allValues
+        .filter((sublayer) => sublayer.length)
+        .map((sublayerValues) =>
+          ckmeans(
+            sublayerValues,
+            Math.min(sublayerValues.length, COLOR_RAMP_BIVARIATE_NUM_STEPS)
+          ).map((step) => step[0])
+        )
       return steps
     }
 
@@ -137,7 +136,7 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer<
         if (!values || !values.length || !Array.isArray(values)) {
           return []
         }
-        return values.filter(filterElementByPercentOfIndex)
+        return values.filter((d, i) => filterCells(d, i, minVisibleValue, maxVisibleValue))
       })
     )
 
@@ -173,12 +172,6 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer<
     colorRanges: FourwingsTileLayerColorRange
   ): FourwinsTileLayerScale[] => {
     if (this.props.comparisonMode === FourwingsComparisonMode.Bivariate) {
-      console.log('colorRanges:', colorRanges)
-      console.log('colorDomain:', colorDomain)
-      // return colorRanges.map((cr) =>
-      //   scaleLinear(colorDomain as number[], cr as string[]).clamp(true)
-      // )
-
       return (colorDomain as number[][]).map((cd, i) => {
         return scaleLinear(cd, colorRanges[i] as string[]).clamp(true)
       })
@@ -306,14 +299,25 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer<
   }
 
   updateState({ props, oldProps }: UpdateParameters<this>) {
-    const { startTime, endTime, availableIntervals } = props
+    const {
+      startTime,
+      endTime,
+      availableIntervals,
+      comparisonMode,
+      minVisibleValue,
+      maxVisibleValue,
+    } = props
     const { tilesCache, colorRanges, colorDomain } = this.state as FourwingsTileLayerState
     const newSublayerColorRanges = this._getColorRanges()
     const sublayersHaveNewColors = colorRanges.join() !== newSublayerColorRanges.join()
-    const newMode = oldProps.comparisonMode && props.comparisonMode !== oldProps.comparisonMode
-
-    if (sublayersHaveNewColors || newMode) {
-      const newColorDomain = newMode ? this._calculateColorDomain() : colorDomain
+    const newMode = oldProps.comparisonMode && comparisonMode !== oldProps.comparisonMode
+    const newVisibleValueLimits =
+      (oldProps.minVisibleValue && minVisibleValue !== oldProps.minVisibleValue) ||
+      (oldProps.maxVisibleValue && maxVisibleValue !== oldProps.maxVisibleValue)
+
+    if (sublayersHaveNewColors || newMode || newVisibleValueLimits) {
+      const newColorDomain =
+        newMode || newVisibleValueLimits ? this._calculateColorDomain() : colorDomain
       const scales = this._getColorScales(newColorDomain, newSublayerColorRanges)
       this.setState({ colorRanges: newSublayerColorRanges, colorDomain: newColorDomain, scales })
     }

libs/deck-layers/src/layers/fourwings/fourwings.utils.ts

@@ -335,7 +335,9 @@ export function getIntervalFrames({
   return { interval, tileStartFrame, startFrame, endFrame }
 }
 
-export function filterElementByPercentOfIndex(value: any, index: number) {
+export function filterCells(value: any, index: number, minValue?: number, maxValue?: number) {
   // Select only 5% of elements
-  return value && index % 20 === 1
+  return (
+    value && index % 20 === 1 && (!minValue || value > minValue) && (!maxValue || value < maxValue)
+  )
 }

libs/deck-layers/src/utils/colorRamps.ts

@@ -117,7 +117,7 @@ export const getBlend = (color1: RGBA, color2: RGBA) => {
   return normal({ ...hexToRgb(BLEND_BACKGROUND), a: 1 }, screen(color1 as RGBA, color2 as RGBA))
 }
 
-export const HEATMAP_COLORS_BY_ID = {
+export const HEATMAP_COLORS_BY_ID: Record<ColorRampId, string> = {
   teal: '#00FFBC',
   magenta: '#FF64CE',
   lilac: '#9CA4FF',
@@ -141,25 +141,16 @@ export const TIME_COMPARE_COLOR_RAMP = [
   '#FF677D',
 ]
 
-export const HEATMAP_COLOR_RAMPS: Record<ColorRampsIds, string[]> = {
-  teal: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.teal),
-  teal_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.teal),
-  magenta: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.magenta),
-  magenta_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.magenta),
-  lilac: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.lilac),
-  lilac_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.lilac),
-  salmon: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.salmon),
-  salmon_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.salmon),
-  sky: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.sky),
-  sky_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.sky),
-  red: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.red),
-  red_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.red),
-  yellow: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.yellow),
-  yellow_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.yellow),
-  green: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.green),
-  green_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.green),
-  orange: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.orange),
-  orange_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.orange),
-  bathymetry: getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID.bathymetry).reverse(),
-  bathymetry_toWhite: getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID.bathymetry).reverse(),
+export const getColorRamp = ({
+  rampId,
+  whiteEnd = false,
+}: {
+  rampId: ColorRampId
+  whiteEnd?: boolean
+}) => {
+  const ramp = whiteEnd
+    ? getMixedOpacityToWhiteColorRamp(HEATMAP_COLORS_BY_ID[rampId])
+    : getColorRampByOpacitySteps(HEATMAP_COLORS_BY_ID[rampId])
+  if (rampId === 'bathymetry') ramp.reverse()
+  return ramp
 }