add support for fourwings cluster filters

apps/fishing-map/features/dataviews/dataviews.mock.ts

@@ -1,5 +1,36 @@
-import { Dataview } from '@globalfishingwatch/api-types'
+import { Dataview, DataviewType, DataviewCategory, EndpointId } from '@globalfishingwatch/api-types'
+import { CLUSTER_ENCOUNTER_EVENTS_DATAVIEW_SLUG } from 'data/workspaces'
 
-const dataviews: Dataview[] = []
+const dataviews: Dataview[] = [
+  {
+    id: 11111111,
+    name: 'Encounter cluster events pipe 3',
+    slug: CLUSTER_ENCOUNTER_EVENTS_DATAVIEW_SLUG,
+    app: 'fishing-map',
+    config: {
+      type: 'FOURWINGS_TILE_CLUSTER',
+      color: '#FAE9A0',
+    },
+    datasetsConfig: [
+      {
+        filters: {
+          'encounter-types': [
+            'CARRIER-FISHING',
+            // 'FISHING-CARRIER',
+            'FISHING-SUPPORT',
+            // 'SUPPORT-FISHING',
+          ],
+        },
+        params: [],
+        endpoint: 'events-cluster-tiles',
+        datasetId: 'public-global-encounters-events:v3.0',
+      },
+    ],
+    description: 'Encounter cluster events',
+    createdAt: '2024-05-16T08:21:11.723Z',
+    updatedAt: '2024-05-16T08:21:11.723Z',
+    category: DataviewCategory.Events,
+  },
+]
 
 export default dataviews

libs/api-types/src/dataviews.ts

@@ -79,7 +79,7 @@ export interface DataviewConfig<Type = DataviewType> {
   /** String encoded for url from filters Record */
   filter?: string
   /** Record with id filter as key and filters as values */
-  filters?: Record<string, any>
+  filters?: DataviewDatasetFilter
   'vessel-groups'?: string[]
   filterOperators?: Record<string, FilterOperator>
   /** Min value for filters in environmental layers to perform frontend data filtering */
@@ -140,12 +140,14 @@ export interface DataviewDatasetConfigParam {
   value: string | number | boolean | string[] | number[]
 }
 
+export type DataviewDatasetFilter = Record<string, any>
 export type DatasetsMigration = Record<string, string>
 export interface DataviewDatasetConfig {
   datasetId: string
   endpoint: string
   params: DataviewDatasetConfigParam[]
   query?: DataviewDatasetConfigParam[]
+  filters?: DataviewDatasetFilter
   metadata?: Record<string, any>
 }
 

libs/datasets-client/src/datasets.utils.ts

@@ -25,7 +25,7 @@ export function getUserDataviewDataset(dataview?: Dataview | UrlDataviewInstance
   ) as Dataset
 }
 
-export const getDatasetsExtent = (
+export const getDatasetsExtent = <Format = 'string' | 'number'>(
   datasets: Dataset[] | undefined,
   { format }: { format: 'isoString' | 'timestamp' } = { format: 'isoString' }
 ) => {
@@ -50,5 +50,5 @@ export const getDatasetsExtent = (
     extentEnd = format === 'isoString' ? extentEndDate.toISO() : extentEndDate.toMillis()
   }
 
-  return { extentStart: extentStart as string | number, extentEnd: extentEnd as string | number }
+  return { extentStart: extentStart as Format, extentEnd: extentEnd as Format }
 }

libs/dataviews-client/src/resolve-dataviews.ts

@@ -324,12 +324,14 @@ export const resolveDataviewDatasetResource = (
   return resolveDataviewDatasetResources(dataview, datasetTypeOrId)[0] || ({} as Resource)
 }
 
-export function getDataviewSqlFiltersResolved(dataview: UrlDataviewInstance) {
-  if (!dataview.config?.filters) {
+export function getDataviewSqlFiltersResolved(dataview: DataviewInstance | UrlDataviewInstance) {
+  const datasetsConfigFilters = (dataview.datasetsConfig || [])?.reduce((acc, datasetConfig) => {
+    return { ...acc, ...(datasetConfig.filters || {}) }
+  }, {} as Record<string, any>)
+  const filters = { ...datasetsConfigFilters, ...(dataview.config?.filters || {}) }
+  if (!Object.keys(filters).length) {
     return ''
   }
-  const { filters, filterOperators } = dataview.config
-
   const sqlFilters = Object.keys(filters)
     .filter((key) => key !== 'vessel-groups')
     .flatMap((filterKey) => {
@@ -363,7 +365,7 @@ export function getDataviewSqlFiltersResolved(dataview: UrlDataviewInstance) {
           return `${queryFilterKey} <= ${maxValue}`
         }
       }
-      const filterOperator = filterOperators?.[filterKey] || INCLUDE_FILTER_ID
+      const filterOperator = dataview.config?.filterOperators?.[filterKey] || INCLUDE_FILTER_ID
       const hasNumericFilterValues = filterValues.every((v: any) => isNumeric(v))
       const query = `${queryFilterKey} ${FILTER_OPERATOR_SQL[filterOperator]} (${filterValues
         .map((f: string) => (hasNumericFilterValues ? `${f}` : `'${f}'`))

libs/deck-layer-composer/src/resolvers/clusters.ts

@@ -1,9 +1,38 @@
-import { UrlDataviewInstance } from '@globalfishingwatch/dataviews-client'
+import { uniqBy } from 'es-toolkit'
+import { DateTime } from 'luxon'
+import {
+  getDataviewSqlFiltersResolved,
+  UrlDataviewInstance,
+} from '@globalfishingwatch/dataviews-client'
 import { FourwingsClustersLayerProps, getUTCDateTime } from '@globalfishingwatch/deck-layers'
 import { getDatasetsExtent, resolveEndpoint } from '@globalfishingwatch/datasets-client'
-import { EndpointId } from '@globalfishingwatch/api-types'
+import { DataviewDatasetConfig, EndpointId } from '@globalfishingwatch/api-types'
 import { DeckResolverFunction, ResolverGlobalConfig } from './types'
 
+const getDateRangeQuery = ({
+  startTime,
+  endTime,
+  extentStart,
+  extentEnd,
+}: {
+  startTime: number
+  endTime: number
+  extentStart?: number
+  extentEnd?: number
+}) => {
+  const start = extentStart && extentStart > startTime ? extentStart : startTime
+  const end =
+    extentEnd && extentEnd < endTime
+      ? DateTime.fromMillis(extentEnd).plus({ day: 1 }).toMillis()
+      : endTime
+  const startIso = getUTCDateTime(start < end ? start : end)
+    .startOf('hour')
+    .toISO()
+  const endIso = getUTCDateTime(end).startOf('hour').toISO()
+
+  return `${startIso},${endIso}`
+}
+
 export const resolveDeckFourwingsClustersLayerProps: DeckResolverFunction<
   FourwingsClustersLayerProps
 > = (
@@ -12,34 +41,56 @@ export const resolveDeckFourwingsClustersLayerProps: DeckResolverFunction<
 ): FourwingsClustersLayerProps => {
   const startTime = start ? getUTCDateTime(start).toMillis() : 0
   const endTime = end ? getUTCDateTime(end).toMillis() : Infinity
-  const { extentStart, extentEnd } = getDatasetsExtent(dataview.datasets, { format: 'timestamp' })
+  const { extentStart, extentEnd } = getDatasetsExtent<number>(dataview.datasets, {
+    format: 'timestamp',
+  })
 
   const dataset = dataview.datasets?.[0]
+  const dataviewDatasetConfig = dataview.datasetsConfig?.[0] || ({} as DataviewDatasetConfig)
+  const datasetId = dataviewDatasetConfig.datasetId || dataset?.id
+
+  if (!dataset || !datasetId) {
+    console.warn('No datasetId found for dataview', dataview)
+    return {} as FourwingsClustersLayerProps
+  }
 
-  const tilesUrl = dataset
-    ? resolveEndpoint(
-        dataset,
+  const datasetConfig = {
+    datasetId: datasetId,
+    endpoint: dataviewDatasetConfig.endpoint || EndpointId.ClusterTiles,
+    params: uniqBy(
+      [...(dataviewDatasetConfig.params || []), { id: 'type', value: 'heatmap' }],
+      (p) => p.id
+    ),
+    query: uniqBy(
+      [
+        ...(dataviewDatasetConfig.query || []),
+        { id: 'format', value: '4WINGS' },
+        { id: 'temporal-aggregation', value: true },
+        { id: 'datasets', value: datasetId },
+        { id: 'filters', value: getDataviewSqlFiltersResolved(dataview) },
         {
-          datasetId: dataset.id,
-          endpoint: EndpointId.ClusterTiles,
-          params: [{ id: 'type', value: 'heatmap' }],
+          id: 'date-range',
+          value: getDateRangeQuery({
+            startTime,
+            endTime,
+            extentStart,
+            extentEnd,
+          }),
         },
-        { absolute: true }
-      )
-    : undefined
+      ],
+      (p) => p.id
+    ),
+  } as DataviewDatasetConfig
 
   return {
     id: dataview.id,
     category: dataview.category!,
     subcategory: dataview.config?.type!,
     datasetId: dataset?.id || '',
     color: dataview.config?.color || '',
-    filters: dataview.config?.filter || '',
     startTime,
     endTime,
     visible: dataview.config?.visible ?? true,
-    tilesUrl: tilesUrl || '',
-    extentStart: extentStart as number,
-    extentEnd: extentEnd as number,
+    tilesUrl: resolveEndpoint(dataset, datasetConfig, { absolute: true }) || '',
   }
 }

libs/deck-layers/src/layers/fourwings/clusters/FourwingsClustersLayer.ts

@@ -8,9 +8,7 @@ import {
   PickingInfo,
 } from '@deck.gl/core'
 import { TileLayer, TileLayerProps } from '@deck.gl/geo-layers'
-import { stringify } from 'qs'
 import { Tile2DHeader, TileLoadProps } from '@deck.gl/geo-layers/dist/tileset-2d'
-import { DateTime } from 'luxon'
 import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers'
 import Supercluster from 'supercluster'
 import { ScalePower, scaleSqrt } from 'd3-scale'
@@ -22,7 +20,6 @@ import {
   DEFAULT_BACKGROUND_COLOR,
   DEFAULT_LINE_COLOR,
   getLayerGroupOffset,
-  getUTCDateTime,
   GFWMVTLoader,
   hexToDeckColor,
   LayerGroup,
@@ -217,38 +214,6 @@ export class FourwingsClustersLayer extends CompositeLayer<
     return this._fetch(url!, { signal: tile.signal, tile })
   }
 
-  _getDataUrl() {
-    const { startTime, endTime, datasetId, filters, extentStart, extentEnd } = this.props
-
-    const start = extentStart && extentStart > startTime ? extentStart : startTime
-    const end =
-      extentEnd && extentEnd < endTime
-        ? DateTime.fromMillis(extentEnd).plus({ day: 1 }).toMillis()
-        : endTime
-    const startIso = getUTCDateTime(start < end ? start : end)
-      .startOf('hour')
-      .toISO()
-    const endIso = getUTCDateTime(end).startOf('hour').toISO()
-    const params = {
-      datasets: [datasetId],
-      ...(filters && { filters: [filters] }),
-      format: '4WINGS',
-      'temporal-aggregation': true,
-      'date-range': `${startIso},${endIso}`,
-    }
-
-    let tilesUrl = HEATMAP_API_TILES_URL
-    try {
-      const { origin, pathname } = new URL(this.props.tilesUrl)
-      tilesUrl = origin + pathname
-    } catch (e) {
-      console.warn(e)
-    }
-    const baseUrl = GFWAPI.generateUrl(tilesUrl, { absolute: true })
-
-    return `${baseUrl}?${stringify(params)}`
-  }
-
   _getPosition = (d: FourwingsClusterFeature) => {
     return d.geometry.coordinates as [number, number]
   }
@@ -268,12 +233,12 @@ export class FourwingsClustersLayer extends CompositeLayer<
   }
 
   renderLayers(): Layer<{}> | LayersList | null {
-    const { color } = this.props
+    const { color, tilesUrl } = this.props
     const { clusters, points, radiusScale } = this.state
     return [
       new TileLayer<FourwingsPointFeature, any>(this.props, {
         id: `${this.props.id}-tiles`,
-        data: this._getDataUrl(),
+        data: tilesUrl,
         maxZoom: POSITIONS_VISUALIZATION_MAX_ZOOM,
         binary: false,
         loaders: [GFWMVTLoader],

libs/deck-layers/src/layers/fourwings/clusters/fourwings-clusters.types.ts

@@ -14,13 +14,10 @@ export type FourwingsClustersLayerProps = DeckLayerProps<{
   endTime: number
   color: string
   datasetId: string
-  filters: string
   eventType?: FourwingsClusterEventType
   maxClusterZoom?: number
   tilesUrl: string
   visible: boolean
-  extentStart?: number
-  extentEnd?: number
 }>
 
 export type FourwingsClusterProperties = {