From 9a9f465fe19a574b146e9f07c290b8aab5b97e16 Mon Sep 17 00:00:00 2001 From: j8seangel Date: Mon, 25 Mar 2024 15:10:00 +0100 Subject: [PATCH 1/5] avoid empty dataviews to merge warning --- .../features/dataviews/selectors/dataviews.selectors.ts | 4 ++-- .../features/timebar/TimebarActivityGraph.hooks.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/fishing-map/features/dataviews/selectors/dataviews.selectors.ts b/apps/fishing-map/features/dataviews/selectors/dataviews.selectors.ts index 4b89fbfd31..918b61654d 100644 --- a/apps/fishing-map/features/dataviews/selectors/dataviews.selectors.ts +++ b/apps/fishing-map/features/dataviews/selectors/dataviews.selectors.ts @@ -195,7 +195,7 @@ export const selectActiveActivityDataviews = createSelector( export const selectActivityMergedDataviewId = createSelector( [selectActiveActivityDataviews], (dataviews): string => { - return getMergedDataviewId(dataviews) + return dataviews?.length ? getMergedDataviewId(dataviews) : '' } ) @@ -219,7 +219,7 @@ export const selectActiveDetectionsDataviews = createSelector( export const selectDetectionsMergedDataviewId = createSelector( [selectActiveDetectionsDataviews], (dataviews): string => { - return getMergedDataviewId(dataviews) + return dataviews?.length ? getMergedDataviewId(dataviews) : '' } ) diff --git a/apps/fishing-map/features/timebar/TimebarActivityGraph.hooks.ts b/apps/fishing-map/features/timebar/TimebarActivityGraph.hooks.ts index 6005dd174b..164063bd73 100644 --- a/apps/fishing-map/features/timebar/TimebarActivityGraph.hooks.ts +++ b/apps/fishing-map/features/timebar/TimebarActivityGraph.hooks.ts @@ -28,7 +28,7 @@ export const useHeatmapActivityGraph = () => { const start = getUTCDate(timerange.start).getTime() const end = getUTCDate(timerange.end).getTime() const chunk = getChunk(start, end) - const id = getMergedDataviewId(dataviews) + const id = dataviews?.length ? getMergedDataviewId(dataviews) : '' const fourwingsActivityLayer = useGetDeckLayer(id) const { loaded, instance } = fourwingsActivityLayer || {} const heatmapActivity = useMemo(() => { From 569c9e2d776b53e823cecc559151442114d264dd Mon Sep 17 00:00:00 2001 From: j8seangel Date: Mon, 25 Mar 2024 15:12:40 +0100 Subject: [PATCH 2/5] rename GFWContextLoader to GFWMVTLoader --- libs/deck-layers/src/layers/context/ContextLayer.ts | 4 ++-- libs/deck-layers/src/layers/context/EEZLayer.ts | 6 +++--- libs/deck-layers/src/utils/loaders.ts | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libs/deck-layers/src/layers/context/ContextLayer.ts b/libs/deck-layers/src/layers/context/ContextLayer.ts index 945931328f..54ec55d1ca 100644 --- a/libs/deck-layers/src/layers/context/ContextLayer.ts +++ b/libs/deck-layers/src/layers/context/ContextLayer.ts @@ -10,7 +10,7 @@ import { LayerGroup, getLayerGroupOffset, getPickedFeatureToHighlight, - GFWContextLoader, + GFWMVTLoader, getMVTSublayerProps, } from '../../utils' import { API_PATH } from './context.config' @@ -57,7 +57,7 @@ export class ContextLayer extends CompositeLayer< new TileLayer({ id: `${this.id}-base-layer`, data: `${API_PATH}/${this.props.datasetId}/context-layers/{z}/{x}/{y}`, - loaders: [GFWContextLoader], + loaders: [GFWMVTLoader], maxRequests: 100, debounceTime: 500, renderSubLayers: (props: any) => { diff --git a/libs/deck-layers/src/layers/context/EEZLayer.ts b/libs/deck-layers/src/layers/context/EEZLayer.ts index 68b7b95473..06ded09b17 100644 --- a/libs/deck-layers/src/layers/context/EEZLayer.ts +++ b/libs/deck-layers/src/layers/context/EEZLayer.ts @@ -7,7 +7,7 @@ import { hexToDeckColor, LayerGroup, getLayerGroupOffset, - GFWContextLoader, + GFWMVTLoader, getMVTSublayerProps, } from '../../utils' import { ContextLayer } from './ContextLayer' @@ -51,7 +51,7 @@ export class EEZLayer extends ContextLayer { new TileLayer({ id: `${this.id}-boundaries-layer`, data: `${API_PATH}/${this.props.boundariesDatasetId}/context-layers/{z}/{x}/{y}`, - loaders: [GFWContextLoader], + loaders: [GFWMVTLoader], maxRequests: 100, debounceTime: 500, renderSubLayers: (props: any) => { @@ -77,7 +77,7 @@ export class EEZLayer extends ContextLayer { new TileLayer({ id: `${this.id}-base-layer`, data: `${API_PATH}/${this.props.areasDatasetId}/context-layers/{z}/{x}/{y}`, - loaders: [GFWContextLoader], + loaders: [GFWMVTLoader], maxRequests: 100, debounceTime: 500, renderSubLayers: (props: any) => { diff --git a/libs/deck-layers/src/utils/loaders.ts b/libs/deck-layers/src/utils/loaders.ts index 111e90df03..26f25e7cc4 100644 --- a/libs/deck-layers/src/utils/loaders.ts +++ b/libs/deck-layers/src/utils/loaders.ts @@ -1,6 +1,7 @@ import { MVTLoader } from '@loaders.gl/mvt' -export const GFWContextLoader = { +export const GFWMVTLoader = { ...MVTLoader, + // TODO: match api response with standard to avoid this override mimeTypes: [...MVTLoader.mimeTypes, 'application/octet-stream'], } From 2475a8717018dc326bcab24b8095e3dbb84e4c42 Mon Sep 17 00:00:00 2001 From: j8seangel Date: Mon, 25 Mar 2024 15:35:28 +0100 Subject: [PATCH 3/5] deck heatmap static layer --- .../src/features/filter-features-by-bounds.ts | 4 +- .../src/resolvers/dataviews.ts | 21 +- .../src/resolvers/fourwings.ts | 8 +- .../src/resolvers/index.ts | 5 +- .../fourwings/FourwingsHeatmapStaticLayer.ts | 241 ++++++++++++++++++ .../fourwings/FourwingsHeatmapTileLayer.ts | 8 +- .../src/layers/fourwings/FourwingsLayer.ts | 32 ++- .../fourwings/FourwingsPositionsTileLayer.ts | 13 +- .../src/layers/fourwings/fourwings.config.ts | 6 +- .../src/layers/fourwings/fourwings.types.ts | 11 +- .../src/layers/fourwings/fourwings.utils.ts | 6 +- libs/deck-loaders/src/fourwings/lib/types.ts | 6 +- 12 files changed, 319 insertions(+), 42 deletions(-) create mode 100644 libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts diff --git a/libs/data-transforms/src/features/filter-features-by-bounds.ts b/libs/data-transforms/src/features/filter-features-by-bounds.ts index f82bbf947b..a0db108baf 100644 --- a/libs/data-transforms/src/features/filter-features-by-bounds.ts +++ b/libs/data-transforms/src/features/filter-features-by-bounds.ts @@ -1,5 +1,5 @@ import type { GeoJSONFeature } from '@globalfishingwatch/maplibre-gl' -import type { FourWingsFeature } from '@globalfishingwatch/deck-loaders' +import type { FourWingsFeature, FourWingsStaticFeature } from '@globalfishingwatch/deck-loaders' export interface Bounds { north: number @@ -9,7 +9,7 @@ export interface Bounds { } export const filterFeaturesByBounds = ( - features: GeoJSONFeature[] | FourWingsFeature[], + features: GeoJSONFeature[] | FourWingsFeature[] | FourWingsStaticFeature[], bounds: Bounds ) => { if (!bounds) { diff --git a/libs/deck-layer-composer/src/resolvers/dataviews.ts b/libs/deck-layer-composer/src/resolvers/dataviews.ts index d2e82b4c2a..6ea8fb6fea 100644 --- a/libs/deck-layer-composer/src/resolvers/dataviews.ts +++ b/libs/deck-layer-composer/src/resolvers/dataviews.ts @@ -72,6 +72,10 @@ function isEnvironmentalDataview(dataview: UrlDataviewInstance) { ) } +function isHeatmapStaticDataview(dataview: UrlDataviewInstance) { + return dataview.config?.type === DataviewType.HeatmapStatic +} + function isTrackDataview(dataview: UrlDataviewInstance) { return ( dataview.category === DataviewCategory.Vessels && dataview.config?.type === DataviewType.Track @@ -224,7 +228,7 @@ export function getDataviewsResolved( const { activityDataviews, detectionDataviews, - environmentalDataviews, + fourwingsDataviews, trackDataviews, otherDataviews, } = dataviews.reduce( @@ -233,8 +237,8 @@ export function getDataviewsResolved( acc.activityDataviews.push(dataview) } else if (isDetectionsDataview(dataview)) { acc.detectionDataviews.push(dataview) - } else if (isEnvironmentalDataview(dataview)) { - acc.environmentalDataviews.push(dataview) + } else if (isEnvironmentalDataview(dataview) || isHeatmapStaticDataview(dataview)) { + acc.fourwingsDataviews.push(dataview) } else if (isTrackDataview(dataview)) { acc.trackDataviews.push(dataview) } else { @@ -245,14 +249,14 @@ export function getDataviewsResolved( { activityDataviews: [] as UrlDataviewInstance[], detectionDataviews: [] as UrlDataviewInstance[], - environmentalDataviews: [] as UrlDataviewInstance[], + fourwingsDataviews: [] as UrlDataviewInstance[], trackDataviews: [] as UrlDataviewInstance[], otherDataviews: [] as UrlDataviewInstance[], } ) const singleHeatmapDataview = - [...activityDataviews, ...detectionDataviews, ...environmentalDataviews].length === 1 + [...activityDataviews, ...detectionDataviews, ...fourwingsDataviews].length === 1 const activityComparisonMode = activityDataviews.every((dataview) => params.bivariateDataviews?.includes(dataview.id) ) @@ -280,17 +284,18 @@ export function getDataviewsResolved( colorRampWhiteEnd: singleHeatmapDataview, }) : [] - const environmentalDataviewsParsed = environmentalDataviews.flatMap( + const fourwingsDataviewsParsed = fourwingsDataviews.flatMap( (d) => getFourwingsDataviewsResolved(d, { - colorRampWhiteEnd: singleHeatmapDataview, + colorRampWhiteEnd: + d.config?.type === DataviewType.HeatmapStatic ? false : singleHeatmapDataview, }) || [] ) const dataviewsMerged = [ ...otherDataviews, ...mergedActivityDataview, ...mergedDetectionsDataview, - ...environmentalDataviewsParsed, + ...fourwingsDataviewsParsed, ...trackDataviews, ] return dataviewsMerged diff --git a/libs/deck-layer-composer/src/resolvers/fourwings.ts b/libs/deck-layer-composer/src/resolvers/fourwings.ts index 042fa0cb11..44cd21f7d2 100644 --- a/libs/deck-layer-composer/src/resolvers/fourwings.ts +++ b/libs/deck-layer-composer/src/resolvers/fourwings.ts @@ -10,7 +10,12 @@ import { TIME_COMPARISON_NOT_SUPPORTED_INTERVALS, getUTCDateTime, } from '@globalfishingwatch/deck-layers' -import { DatasetTypes, DataviewCategory, EndpointId } from '@globalfishingwatch/api-types' +import { + DatasetTypes, + DataviewCategory, + DataviewType, + EndpointId, +} from '@globalfishingwatch/api-types' import { ColorRampId } from '@globalfishingwatch/deck-layers' import { resolveEndpoint } from '@globalfishingwatch/datasets-client' import { getDataviewAvailableIntervals } from './dataviews' @@ -90,6 +95,7 @@ export const resolveDeckFourwingsLayerProps = ( id: dataview.id, minFrame: startTime, maxFrame: endTime, + static: dataview.config?.type === DataviewType.HeatmapStatic, resolution, sublayers, comparisonMode, diff --git a/libs/deck-layer-composer/src/resolvers/index.ts b/libs/deck-layer-composer/src/resolvers/index.ts index 7fe39bce33..fe9c48a4c3 100644 --- a/libs/deck-layer-composer/src/resolvers/index.ts +++ b/libs/deck-layer-composer/src/resolvers/index.ts @@ -33,7 +33,10 @@ export const dataviewToDeckLayer = ( const deckLayerProps = resolveDeckBasemapLayerProps(dataview) return new BaseMapLayer(deckLayerProps) } - if (dataview.config?.type === DataviewType.HeatmapAnimated) { + if ( + dataview.config?.type === DataviewType.HeatmapAnimated || + dataview.config?.type === DataviewType.HeatmapStatic + ) { const deckLayerProps = resolveDeckFourwingsLayerProps(dataview, globalConfig, interactions) const layer = new FourwingsLayer(deckLayerProps) return layer diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts new file mode 100644 index 0000000000..5fd262e548 --- /dev/null +++ b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts @@ -0,0 +1,241 @@ +import { CompositeLayer, Layer, LayerContext, LayersList, DefaultProps, Color } from '@deck.gl/core' +import { scaleLinear } from 'd3-scale' +import { MVTLayer, TileLayer, TileLayerProps } from '@deck.gl/geo-layers' +import { ckmeans } from 'simple-statistics' +import { debounce } from 'lodash' +import { Tile2DHeader } from '@deck.gl/geo-layers/dist/tileset-2d' +import { PathLayer } from '@deck.gl/layers' +import { stringify } from 'qs' +import { PathStyleExtension } from '@deck.gl/extensions' +import { Feature, Geometry } from 'geojson' +import { + FourWingsFeature, + FourWingsStaticFeature, + FourWingsStaticFeatureProperties, +} from '@globalfishingwatch/deck-loaders' +import { + HEATMAP_COLOR_RAMPS, + rgbaStringToComponents, + ColorRampsIds, +} from '@globalfishingwatch/layer-composer' +import { GFWAPI } from '@globalfishingwatch/api-client' +import { filterFeaturesByBounds } from '@globalfishingwatch/data-transforms' +import { COLOR_RAMP_DEFAULT_NUM_STEPS } from '../../utils/colorRamps' +import { + COLOR_HIGHLIGHT_LINE, + GFWMVTLoader, + LayerGroup, + deckToRgbaColor, + getLayerGroupOffset, +} from '../../utils' +import { + EMPTY_CELL_COLOR, + filterElementByPercentOfIndex, + getRoundedDateFromTS, +} from './fourwings.utils' +import { + HEATMAP_API_TILES_URL, + HEATMAP_STATIC_ID, + MAX_RAMP_VALUES_PER_TILE, +} from './fourwings.config' +import { + ColorRange, + FourwingsHeatmapTileLayerProps, + FourwingsTileLayerState, + FourwingsComparisonMode, + FourwingsAggregationOperation, + FourwinsTileLayerScale, +} from './fourwings.types' + +const defaultProps: DefaultProps = { + maxRequests: 100, + debounceTime: 500, + comparisonMode: FourwingsComparisonMode.Compare, + aggregationOperation: FourwingsAggregationOperation.Sum, + tilesUrl: HEATMAP_API_TILES_URL, + resolution: 'default', +} + +export class FourwingsHeatmapStaticLayer extends CompositeLayer< + FourwingsHeatmapTileLayerProps & TileLayerProps +> { + static layerName = 'FourwingsHeatmapStaticLayer' + static defaultProps = defaultProps + initialBinsLoad = false + scale: typeof scaleLinear | undefined = undefined + + initializeState(context: LayerContext) { + super.initializeState(context) + this.state = { + colorDomain: [], + colorRanges: this._getColorRanges(), + scale: scaleLinear([], []), + } + } + + _getState() { + return this.state as FourwingsTileLayerState + } + + _getColorRanges = () => { + return this.props.sublayers.map( + ({ colorRamp }) => + HEATMAP_COLOR_RAMPS[colorRamp as ColorRampsIds].map((c) => + rgbaStringToComponents(c) + ) as ColorRange + ) + } + + _calculateColorDomain = () => { + // TODO use to get the real bin value considering the NO_DATA_VALUE and negatives + // NO_DATA_VALUE = 0 + // SCALE_VALUE = 0.01 + // OFFSET_VALUE = 0 + const currentZoomData = this.getData() + if (!currentZoomData.length) { + return this.getColorDomain() + } + const dataSample = + currentZoomData.length > MAX_RAMP_VALUES_PER_TILE + ? currentZoomData.filter(filterElementByPercentOfIndex) + : currentZoomData + + const allValues = dataSample.flatMap((feature) => feature.properties?.count) + + const steps = ckmeans(allValues, Math.min(allValues.length, COLOR_RAMP_DEFAULT_NUM_STEPS)).map( + (step) => step[0] + ) + return steps + } + + updateColorDomain = debounce(() => { + requestAnimationFrame(() => { + const colorDomain = this._calculateColorDomain() as number[] + const colorRanges = this._getColorRanges()?.[0]?.map((c) => deckToRgbaColor(c)) + this.setState({ colorDomain, scale: scaleLinear(colorDomain, colorRanges) }) + }) + }, 500) + + _onViewportLoad = (tiles: Tile2DHeader[]) => { + this.updateColorDomain() + if (this.props.onViewportLoad) { + this.props.onViewportLoad(tiles) + } + } + + getFillColor = (feature: Feature) => { + if (!this.state.scale) { + return EMPTY_CELL_COLOR + } + if (!feature.properties.count) { + // TODO make the filter for the visible data here + return EMPTY_CELL_COLOR + } + return rgbaStringToComponents( + (this.state.scale as FourwinsTileLayerScale)(feature.properties.count) + ) as Color + } + + renderLayers(): Layer<{}> | LayersList { + const { minFrame, maxFrame, sublayers } = this.props + const { colorDomain, colorRanges, scale } = this.state as FourwingsTileLayerState + const params = { + datasets: sublayers.flatMap((sublayer) => sublayer.datasets), + format: 'MVT', + 'temporal-aggregation': true, + 'date-range': `${getRoundedDateFromTS(minFrame)},${getRoundedDateFromTS(maxFrame)}`, + } + + const baseUrl = GFWAPI.generateUrl(this.props.tilesUrl as string, { absolute: true }) + + const layers: any[] = [ + new MVTLayer(this.props as any, { + id: `${HEATMAP_STATIC_ID}-static`, + data: `${baseUrl}?${stringify(params)}`, + // maxZoom: POSITIONS_VISUALIZATION_MIN_ZOOM, + binary: false, + pickable: true, + loaders: [GFWMVTLoader], + onViewportLoad: this._onViewportLoad, + getPolygonOffset: (params) => getLayerGroupOffset(LayerGroup.Point, params), + getFillColor: this.getFillColor, + updateTriggers: { + getFillColor: [colorDomain, colorRanges, scale], + }, + }), + ] + + const layerHoveredFeature: FourWingsStaticFeature = this.props.hoveredFeatures?.find( + (f) => f.layer?.id === this.root.id + )?.object + + if (layerHoveredFeature) { + layers.push( + new PathLayer( + this.props, + this.getSubLayerProps({ + data: [layerHoveredFeature], + id: `fourwings-cell-highlight`, + widthUnits: 'pixels', + widthMinPixels: 4, + getPath: (d: FourWingsFeature) => d.geometry.coordinates[0], + getColor: COLOR_HIGHLIGHT_LINE, + getOffset: 0.5, + getPolygonOffset: (params: any) => + getLayerGroupOffset(LayerGroup.OutlinePolygonsHighlighted, params), + extensions: [new PathStyleExtension({ offset: true })], + }) + ) + ) + } + + return layers + } + + getLayerInstance() { + const layer = this.getSubLayers()[0] as TileLayer + return layer + } + + getData() { + const layer = this.getLayerInstance() + if (layer) { + const zoom = Math.round(this.context.viewport.zoom) + const offset = this.props.resolution === 'high' ? 1 : 0 + return layer.getSubLayers().flatMap((l: any) => { + if (l.props.tile.zoom === l.props.maxZoom) { + return l.props.data() + } + return l.props.tile.zoom === zoom + offset ? l.props.data : [] + }) as FourWingsStaticFeature[] + } + return [] as FourWingsStaticFeature[] + } + + getViewportData() { + const data = this.getData() + const { viewport } = this.context + const [west, north] = viewport.unproject([0, 0]) + const [east, south] = viewport.unproject([viewport.width, viewport.height]) + if (data?.length) { + const dataFiltered = filterFeaturesByBounds(data, { north, south, west, east }) + return dataFiltered as FourWingsFeature[] + } + return [] + } + + getColorDomain = () => { + return (this.state as FourwingsTileLayerState).colorDomain + } + + getColorRange = () => { + return (this.state as FourwingsTileLayerState).colorRanges + } + + getColorScale = () => { + return { + range: this.getColorRange(), + domain: this.getColorDomain(), + } + } +} diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapTileLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapTileLayer.ts index 48ec978e5a..00b0b4548f 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapTileLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapTileLayer.ts @@ -35,9 +35,9 @@ import { FourwingsHeatmapLayer } from './FourwingsHeatmapLayer' import { HEATMAP_API_TILES_URL, FOURWINGS_MAX_ZOOM, - HEATMAP_ID, PATH_BASENAME, getInterval, + MAX_RAMP_VALUES_PER_TILE, } from './fourwings.config' import { ColorRange, @@ -58,8 +58,6 @@ const defaultProps: DefaultProps = { resolution: 'default', } -const MAX_VALUES_PER_TILE = 1000 - export class FourwingsHeatmapTileLayer extends CompositeLayer< FourwingsHeatmapTileLayerProps & TileLayerProps > { @@ -110,7 +108,7 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer< return this.getColorDomain() } const dataSample = - currentZoomData.length > MAX_VALUES_PER_TILE + currentZoomData.length > MAX_RAMP_VALUES_PER_TILE ? currentZoomData.filter(filterElementByPercentOfIndex) : currentZoomData @@ -318,7 +316,7 @@ export class FourwingsHeatmapTileLayer extends CompositeLayer< return new TileLayer( this.props, this.getSubLayerProps({ - id: `${HEATMAP_ID}-tiles-${resolution}`, + id: `tiles-${resolution}`, tileSize: 512, colorDomain, colorRanges, diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts index 78c1775bd1..7a59b3ffc8 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts @@ -1,14 +1,15 @@ import { Color, CompositeLayer, Layer, LayerContext, LayersList } from '@deck.gl/core' import { TileLayerProps } from '@deck.gl/geo-layers' import { Tile2DHeader } from '@deck.gl/geo-layers/dist/tileset-2d' -import { FourWingsFeature } from '@globalfishingwatch/deck-loaders' import { FourwingsHeatmapTileLayer } from './FourwingsHeatmapTileLayer' +import { FourwingsHeatmapStaticLayer } from './FourwingsHeatmapStaticLayer' +import { FourwingsPositionsTileLayer } from './FourwingsPositionsTileLayer' +import { HEATMAP_ID, HEATMAP_STATIC_ID, POSITIONS_ID } from './fourwings.config' import { - FourwingsPositionsTileLayer, + FourwingsHeatmapTileLayerProps, FourwingsPositionsTileLayerProps, -} from './FourwingsPositionsTileLayer' -import { HEATMAP_ID, POSITIONS_ID } from './fourwings.config' -import { FourwingsHeatmapTileLayerProps, FourwingsVisualizationMode } from './fourwings.types' + FourwingsVisualizationMode, +} from './fourwings.types' export type FourwingsColorRamp = { colorDomain: number[] @@ -44,20 +45,31 @@ export class FourwingsLayer extends CompositeLayer | LayersList { const visualizationMode = this.getMode() const HeatmapLayerClass = this.getSubLayerClass('heatmap', FourwingsHeatmapTileLayer) + const HeatmapStaticLayerClass = this.getSubLayerClass('heatmap', FourwingsHeatmapStaticLayer) const PositionsLayerClass = this.getSubLayerClass('positions', FourwingsPositionsTileLayer) - return visualizationMode === HEATMAP_ID - ? new HeatmapLayerClass( + if (visualizationMode === POSITIONS_ID) { + return new PositionsLayerClass( + this.props, + this.getSubLayerProps({ + id: POSITIONS_ID, + onViewportLoad: this._onViewportLoad, + onTileDataLoading: this._onTileDataLoading, + }) + ) + } + return this.props.static + ? new HeatmapStaticLayerClass( this.props, this.getSubLayerProps({ - id: HEATMAP_ID, + id: HEATMAP_STATIC_ID, onViewportLoad: this._onViewportLoad, onTileDataLoading: this._onTileDataLoading, }) ) - : new PositionsLayerClass( + : new HeatmapLayerClass( this.props, this.getSubLayerProps({ - id: POSITIONS_ID, + id: HEATMAP_ID, onViewportLoad: this._onViewportLoad, onTileDataLoading: this._onTileDataLoading, }) diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsPositionsTileLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsPositionsTileLayer.ts index c88438d19b..bee053a7dd 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsPositionsTileLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsPositionsTileLayer.ts @@ -12,22 +12,15 @@ import { Tile2DHeader } from '@deck.gl/geo-layers/dist/tileset-2d' import { GFWAPI } from '@globalfishingwatch/api-client' import { COLOR_RAMP_DEFAULT_NUM_STEPS } from '@globalfishingwatch/layer-composer' import { getLayerGroupOffset, LayerGroup } from '../../utils' -import { - POSITIONS_API_TILES_URL, - POSITIONS_ID, - POSITIONS_VISUALIZATION_MIN_ZOOM, -} from './fourwings.config' +import { POSITIONS_API_TILES_URL, POSITIONS_VISUALIZATION_MIN_ZOOM } from './fourwings.config' import { getRoundedDateFromTS } from './fourwings.utils' import { FourwingsTileLayerColorDomain, FourwingsTileLayerColorRange, FourwingsTileLayerColorScale, - _FourwingsPositionsTileLayerProps, + FourwingsPositionsTileLayerProps, } from './fourwings.types' -export type FourwingsPositionsTileLayerProps = _FourwingsPositionsTileLayerProps & - Partial - type FourwingsPositionsTileLayerState = { colorScale?: FourwingsTileLayerColorScale allPositions: Feature[] @@ -205,7 +198,7 @@ export class FourwingsPositionsTileLayer extends CompositeLayer< const baseUrl = GFWAPI.generateUrl(this.props.tilesUrl as string, { absolute: true }) return [ new MVTLayer(this.props as any, { - id: `${POSITIONS_ID}-tiles`, + id: `tiles`, data: `${baseUrl}?${stringify(params)}`, minZoom: POSITIONS_VISUALIZATION_MIN_ZOOM, maxZoom: POSITIONS_VISUALIZATION_MIN_ZOOM, diff --git a/libs/deck-layers/src/layers/fourwings/fourwings.config.ts b/libs/deck-layers/src/layers/fourwings/fourwings.config.ts index 6c9ec362f6..d79e285205 100644 --- a/libs/deck-layers/src/layers/fourwings/fourwings.config.ts +++ b/libs/deck-layers/src/layers/fourwings/fourwings.config.ts @@ -7,7 +7,8 @@ import { Chunk } from './fourwings.types' export const IS_PRODUCTION = process.env.NODE_ENV === 'production' export const PATH_BASENAME = process.env.NEXT_PUBLIC_URL || (IS_PRODUCTION ? '/map' : '') -const BASE_API_TILES_URL = `${API_GATEWAY}/${API_VERSION}/4wings/tile/{FOURWINGS_VISUALIZATION_MODE}/{z}/{x}/{y}` +const BASE_API_TILES_URL = + `${API_GATEWAY}/${API_VERSION}/4wings/tile/{FOURWINGS_VISUALIZATION_MODE}/{z}/{x}/{y}` as const export const HEATMAP_API_TILES_URL = BASE_API_TILES_URL.replace( '{FOURWINGS_VISUALIZATION_MODE}', 'heatmap' @@ -18,11 +19,14 @@ export const POSITIONS_API_TILES_URL = BASE_API_TILES_URL.replace( ) export const HEATMAP_ID = 'heatmap' +export const HEATMAP_STATIC_ID = 'heatmap-static' export const POSITIONS_ID = 'positions' export const FOURWINGS_MAX_ZOOM = 12 export const POSITIONS_VISUALIZATION_MIN_ZOOM = 9 +export const MAX_RAMP_VALUES_PER_TILE = 1000 + export const DEFAULT_FOURWINGS_INTERVALS: FourwingsInterval[] = ['HOUR', 'DAY', 'MONTH', 'YEAR'] export const TIME_COMPARISON_NOT_SUPPORTED_INTERVALS: FourwingsInterval[] = ['MONTH', 'YEAR'] diff --git a/libs/deck-layers/src/layers/fourwings/fourwings.types.ts b/libs/deck-layers/src/layers/fourwings/fourwings.types.ts index 7eba465281..e5eb4ae079 100644 --- a/libs/deck-layers/src/layers/fourwings/fourwings.types.ts +++ b/libs/deck-layers/src/layers/fourwings/fourwings.types.ts @@ -1,6 +1,7 @@ import { Color, PickingInfo } from '@deck.gl/core' import { TileLayerProps } from '@deck.gl/geo-layers' import { Tile2DHeader, TileLoadProps } from '@deck.gl/geo-layers/dist/tileset-2d' +import { scaleLinear } from 'd3-scale' import { ColorRampsIds } from '@globalfishingwatch/layer-composer' import { FourWingsFeature, FourwingsInterval } from '@globalfishingwatch/deck-loaders' import { HEATMAP_ID, POSITIONS_ID } from './fourwings.config' @@ -72,6 +73,7 @@ export type GetFillColorParams = { maxIntervalFrame: number comparisonMode?: FourwingsComparisonMode aggregationOperation?: FourwingsAggregationOperation + scale?: typeof scaleLinear } type BaseFourwinsLayerProps = { @@ -97,7 +99,11 @@ export type _FourwingsHeatmapTileLayerProps = BaseFour onTileDataLoading?: (tile: TileLoadProps) => void } +export type FourwingsHeatmapTileLayerProps = _FourwingsHeatmapTileLayerProps & + Partial + export type _FourwingsPositionsTileLayerProps = BaseFourwinsLayerProps & { + static?: boolean highlightedVesselId?: string onDataLoad?: (data: DataT) => void // onColorRampUpdate: (colorRamp: FourwingsColorRamp) => void @@ -107,7 +113,7 @@ export type _FourwingsPositionsTileLayerProps = BaseFourwinsLayerPr onTileDataLoading?: (tile: TileLoadProps) => void } -export type FourwingsHeatmapTileLayerProps = _FourwingsHeatmapTileLayerProps & +export type FourwingsPositionsTileLayerProps = _FourwingsPositionsTileLayerProps & Partial export type FourwingsHeatmapTilesCache = { @@ -124,10 +130,11 @@ export type FourwingsTileLayerColorScale = { export type FourwingsTileLayerColorDomain = number[] | number[][] export type FourwingsTileLayerColorRange = string[] | ColorRange[] +export type FourwinsTileLayerScale = ReturnType> export type FourwingsTileLayerState = { tilesCache: FourwingsHeatmapTilesCache colorDomain: FourwingsTileLayerColorDomain colorRanges: FourwingsTileLayerColorRange comparisonMode?: FourwingsComparisonMode - tiles: Tile2DHeader[] + scale?: FourwinsTileLayerScale } diff --git a/libs/deck-layers/src/layers/fourwings/fourwings.utils.ts b/libs/deck-layers/src/layers/fourwings/fourwings.utils.ts index 165353be8d..a0b9261cf1 100644 --- a/libs/deck-layers/src/layers/fourwings/fourwings.utils.ts +++ b/libs/deck-layers/src/layers/fourwings/fourwings.utils.ts @@ -280,7 +280,7 @@ export const getBivariateValue = (realValues: number[], breaks: number[][]) => { return index + 1 } -const EMPTY_CELL_COLOR: Color = [0, 0, 0, 0] +export const EMPTY_CELL_COLOR: Color = [0, 0, 0, 0] export const chooseColor = ( feature: FourWingsFeature, { @@ -291,6 +291,7 @@ export const chooseColor = ( maxIntervalFrame, aggregationOperation, comparisonMode, + scale, }: GetFillColorParams ): Color => { if (!colorDomain || !colorRanges || !chunk) { @@ -316,6 +317,9 @@ export const chooseColor = ( chosenValueIndex = index } }) + // if (scale) { + // return rgbaStringToComponents(scale(chosenValue)) as Color + // } const colorIndex = (colorDomain as number[]).findIndex((d, i) => (chosenValue as number) <= d || i === colorRanges[0].length - 1 ? i : 0 ) diff --git a/libs/deck-loaders/src/fourwings/lib/types.ts b/libs/deck-loaders/src/fourwings/lib/types.ts index c943cd1988..c1f7406ca9 100644 --- a/libs/deck-loaders/src/fourwings/lib/types.ts +++ b/libs/deck-loaders/src/fourwings/lib/types.ts @@ -50,5 +50,9 @@ export type FourWingsFeatureProperties = { cellId: number cellNum: number } +export type FourWingsStaticFeatureProperties = { + count: number +} -export type FourWingsFeature = Feature +export type FourWingsFeature = Feature +export type FourWingsStaticFeature = FourWingsFeature From 47ff3dec9568d648a0424ead74c40815bbd4eb3b Mon Sep 17 00:00:00 2001 From: satellitestudiodesign Date: Mon, 25 Mar 2024 15:50:34 +0100 Subject: [PATCH 4/5] pr review --- .../layers/fourwings/FourwingsHeatmapStaticLayer.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts index 5fd262e548..c5b2fa75fc 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts @@ -28,11 +28,7 @@ import { deckToRgbaColor, getLayerGroupOffset, } from '../../utils' -import { - EMPTY_CELL_COLOR, - filterElementByPercentOfIndex, - getRoundedDateFromTS, -} from './fourwings.utils' +import { EMPTY_CELL_COLOR, filterElementByPercentOfIndex } from './fourwings.utils' import { HEATMAP_API_TILES_URL, HEATMAP_STATIC_ID, @@ -61,7 +57,6 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< > { static layerName = 'FourwingsHeatmapStaticLayer' static defaultProps = defaultProps - initialBinsLoad = false scale: typeof scaleLinear | undefined = undefined initializeState(context: LayerContext) { @@ -137,13 +132,12 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< } renderLayers(): Layer<{}> | LayersList { - const { minFrame, maxFrame, sublayers } = this.props + const { sublayers } = this.props const { colorDomain, colorRanges, scale } = this.state as FourwingsTileLayerState const params = { datasets: sublayers.flatMap((sublayer) => sublayer.datasets), format: 'MVT', 'temporal-aggregation': true, - 'date-range': `${getRoundedDateFromTS(minFrame)},${getRoundedDateFromTS(maxFrame)}`, } const baseUrl = GFWAPI.generateUrl(this.props.tilesUrl as string, { absolute: true }) @@ -157,7 +151,7 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< pickable: true, loaders: [GFWMVTLoader], onViewportLoad: this._onViewportLoad, - getPolygonOffset: (params) => getLayerGroupOffset(LayerGroup.Point, params), + getPolygonOffset: (params) => getLayerGroupOffset(LayerGroup.HeatmapStatic, params), getFillColor: this.getFillColor, updateTriggers: { getFillColor: [colorDomain, colorRanges, scale], From 2dfee2a70ed07ba37639103fb732b9cceca79af3 Mon Sep 17 00:00:00 2001 From: j8seangel Date: Mon, 25 Mar 2024 16:26:10 +0100 Subject: [PATCH 5/5] min and maxVisible filters in heatmap static --- .../src/resolvers/dataviews.ts | 2 + .../src/resolvers/fourwings.ts | 2 + .../fourwings/FourwingsHeatmapStaticLayer.ts | 78 +++++++++++++------ .../src/layers/fourwings/FourwingsLayer.ts | 2 + .../src/layers/fourwings/fourwings.types.ts | 10 +++ 5 files changed, 71 insertions(+), 23 deletions(-) diff --git a/libs/deck-layer-composer/src/resolvers/dataviews.ts b/libs/deck-layer-composer/src/resolvers/dataviews.ts index 6ea8fb6fea..a57ba69476 100644 --- a/libs/deck-layer-composer/src/resolvers/dataviews.ts +++ b/libs/deck-layer-composer/src/resolvers/dataviews.ts @@ -140,6 +140,8 @@ export function getFourwingsDataviewsResolved( config: { type: fourwingsDataviews[0]?.config?.type, sublayers: fourwingsDataviews.flatMap(getFourwingsDataviewSublayers), + minVisibleValue: fourwingsDataviews[0].config?.minVisibleValue, + maxVisibleValue: fourwingsDataviews[0].config?.maxVisibleValue, colorRampWhiteEnd, visualizationMode, comparisonMode, diff --git a/libs/deck-layer-composer/src/resolvers/fourwings.ts b/libs/deck-layer-composer/src/resolvers/fourwings.ts index 44cd21f7d2..643a7eca4d 100644 --- a/libs/deck-layer-composer/src/resolvers/fourwings.ts +++ b/libs/deck-layer-composer/src/resolvers/fourwings.ts @@ -103,6 +103,8 @@ export const resolveDeckFourwingsLayerProps = ( aggregationOperation, availableIntervals, hoveredFeatures: interactions, + minVisibleValue: dataview.config?.minVisibleValue, + maxVisibleValue: dataview.config?.maxVisibleValue, debug: dataview.config?.debug ?? false, visible: dataview.config?.visible ?? true, colorRampWhiteEnd: dataview.config?.colorRampWhiteEnd ?? false, diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts index c5b2fa75fc..d681e5d674 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsHeatmapStaticLayer.ts @@ -1,4 +1,12 @@ -import { CompositeLayer, Layer, LayerContext, LayersList, DefaultProps, Color } from '@deck.gl/core' +import { + CompositeLayer, + Layer, + LayerContext, + LayersList, + DefaultProps, + Color, + UpdateParameters, +} from '@deck.gl/core' import { scaleLinear } from 'd3-scale' import { MVTLayer, TileLayer, TileLayerProps } from '@deck.gl/geo-layers' import { ckmeans } from 'simple-statistics' @@ -29,24 +37,19 @@ import { getLayerGroupOffset, } from '../../utils' import { EMPTY_CELL_COLOR, filterElementByPercentOfIndex } from './fourwings.utils' -import { - HEATMAP_API_TILES_URL, - HEATMAP_STATIC_ID, - MAX_RAMP_VALUES_PER_TILE, -} from './fourwings.config' +import { HEATMAP_API_TILES_URL, MAX_RAMP_VALUES_PER_TILE } from './fourwings.config' import { ColorRange, FourwingsHeatmapTileLayerProps, FourwingsTileLayerState, - FourwingsComparisonMode, FourwingsAggregationOperation, FourwinsTileLayerScale, + FourwingsHeatmapStaticLayerProps, } from './fourwings.types' -const defaultProps: DefaultProps = { +const defaultProps: DefaultProps = { maxRequests: 100, debounceTime: 500, - comparisonMode: FourwingsComparisonMode.Compare, aggregationOperation: FourwingsAggregationOperation.Sum, tilesUrl: HEATMAP_API_TILES_URL, resolution: 'default', @@ -95,24 +98,35 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< ? currentZoomData.filter(filterElementByPercentOfIndex) : currentZoomData - const allValues = dataSample.flatMap((feature) => feature.properties?.count) + const allValues = dataSample.flatMap((feature) => { + if ( + (this.props.minVisibleValue && feature.properties.count < this.props.minVisibleValue) || + (this.props.maxVisibleValue && feature.properties.count > this.props.maxVisibleValue) + ) { + return [] + } + return feature.properties?.count + }) const steps = ckmeans(allValues, Math.min(allValues.length, COLOR_RAMP_DEFAULT_NUM_STEPS)).map( (step) => step[0] ) + return steps } - updateColorDomain = debounce(() => { - requestAnimationFrame(() => { - const colorDomain = this._calculateColorDomain() as number[] - const colorRanges = this._getColorRanges()?.[0]?.map((c) => deckToRgbaColor(c)) - this.setState({ colorDomain, scale: scaleLinear(colorDomain, colorRanges) }) - }) + _updateColorDomain = () => { + const colorDomain = this._calculateColorDomain() as number[] + const colorRanges = this._getColorRanges()?.[0]?.map((c) => deckToRgbaColor(c)) + this.setState({ colorDomain, scale: scaleLinear(colorDomain, colorRanges) }) + } + + debouncedUpdateColorDomain = debounce(() => { + requestAnimationFrame(this._updateColorDomain) }, 500) _onViewportLoad = (tiles: Tile2DHeader[]) => { - this.updateColorDomain() + this.debouncedUpdateColorDomain() if (this.props.onViewportLoad) { this.props.onViewportLoad(tiles) } @@ -126,13 +140,30 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< // TODO make the filter for the visible data here return EMPTY_CELL_COLOR } - return rgbaStringToComponents( - (this.state.scale as FourwinsTileLayerScale)(feature.properties.count) - ) as Color + if ( + (this.props.minVisibleValue && feature.properties.count < this.props.minVisibleValue) || + (this.props.maxVisibleValue && feature.properties.count > this.props.maxVisibleValue) + ) { + return EMPTY_CELL_COLOR + } + const value = (this.state.scale as FourwinsTileLayerScale)(feature.properties.count) + if (!value) { + return EMPTY_CELL_COLOR + } + return rgbaStringToComponents(value) as Color + } + + updateState({ props, oldProps }: UpdateParameters) { + const { minVisibleValue, maxVisibleValue } = props + const isVisibleValuesChanged = + minVisibleValue !== oldProps.minVisibleValue || maxVisibleValue !== oldProps.maxVisibleValue + if (isVisibleValuesChanged) { + this._updateColorDomain() + } } renderLayers(): Layer<{}> | LayersList { - const { sublayers } = this.props + const { sublayers, resolution, minVisibleValue, maxVisibleValue } = this.props const { colorDomain, colorRanges, scale } = this.state as FourwingsTileLayerState const params = { datasets: sublayers.flatMap((sublayer) => sublayer.datasets), @@ -144,17 +175,18 @@ export class FourwingsHeatmapStaticLayer extends CompositeLayer< const layers: any[] = [ new MVTLayer(this.props as any, { - id: `${HEATMAP_STATIC_ID}-static`, + id: `static-${resolution}`, data: `${baseUrl}?${stringify(params)}`, // maxZoom: POSITIONS_VISUALIZATION_MIN_ZOOM, binary: false, pickable: true, loaders: [GFWMVTLoader], + zoomOffset: resolution === 'high' ? 1 : 0, onViewportLoad: this._onViewportLoad, getPolygonOffset: (params) => getLayerGroupOffset(LayerGroup.HeatmapStatic, params), getFillColor: this.getFillColor, updateTriggers: { - getFillColor: [colorDomain, colorRanges, scale], + getFillColor: [colorDomain, colorRanges, scale, minVisibleValue, maxVisibleValue], }, }), ] diff --git a/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts b/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts index 7a59b3ffc8..e0024300aa 100644 --- a/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts +++ b/libs/deck-layers/src/layers/fourwings/FourwingsLayer.ts @@ -6,6 +6,7 @@ import { FourwingsHeatmapStaticLayer } from './FourwingsHeatmapStaticLayer' import { FourwingsPositionsTileLayer } from './FourwingsPositionsTileLayer' import { HEATMAP_ID, HEATMAP_STATIC_ID, POSITIONS_ID } from './fourwings.config' import { + FourwingsHeatmapStaticLayerProps, FourwingsHeatmapTileLayerProps, FourwingsPositionsTileLayerProps, FourwingsVisualizationMode, @@ -17,6 +18,7 @@ export type FourwingsColorRamp = { } export type FourwingsLayerProps = FourwingsPositionsTileLayerProps & + FourwingsHeatmapStaticLayerProps & FourwingsHeatmapTileLayerProps & { id: string visualizationMode?: FourwingsVisualizationMode diff --git a/libs/deck-layers/src/layers/fourwings/fourwings.types.ts b/libs/deck-layers/src/layers/fourwings/fourwings.types.ts index e5eb4ae079..f9d8da1866 100644 --- a/libs/deck-layers/src/layers/fourwings/fourwings.types.ts +++ b/libs/deck-layers/src/layers/fourwings/fourwings.types.ts @@ -94,6 +94,8 @@ export type _FourwingsHeatmapTileLayerProps = BaseFour availableIntervals?: FourwingsInterval[] resolution?: FourwingsResolution colorRampWhiteEnd?: boolean + minVisibleValue?: number + maxVisibleValue?: number comparisonMode?: FourwingsComparisonMode aggregationOperation?: FourwingsAggregationOperation onTileDataLoading?: (tile: TileLoadProps) => void @@ -102,6 +104,14 @@ export type _FourwingsHeatmapTileLayerProps = BaseFour export type FourwingsHeatmapTileLayerProps = _FourwingsHeatmapTileLayerProps & Partial +export type _FourwingsHeatmapStaticLayerProps = Omit< + _FourwingsHeatmapTileLayerProps, + 'data' | 'availableIntervals' | 'comparisonMode' +> + +export type FourwingsHeatmapStaticLayerProps = _FourwingsHeatmapStaticLayerProps & + Partial + export type _FourwingsPositionsTileLayerProps = BaseFourwinsLayerProps & { static?: boolean highlightedVesselId?: string