From a6b229f98c12f246384031bbea7301546868c043 Mon Sep 17 00:00:00 2001 From: Sandy Spicer Date: Thu, 12 Sep 2024 21:22:58 -0700 Subject: [PATCH] feat: split funnel trends and steps flags (#24949) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/src/lib/constants.tsx | 1 + .../insights/funnels/funnels_query_runner.py | 15 +- .../__snapshots__/test_funnel_trends_udf.ambr | 362 ------------------ .../test/__snapshots__/test_funnel_udf.ambr | 8 +- .../funnels/test/test_funnel_trends.py | 25 +- .../funnels/test/test_funnel_trends_udf.py | 51 ++- .../insights/funnels/test/test_funnel_udf.py | 55 ++- .../hogql_queries/insights/funnels/utils.py | 12 +- .../test/__snapshots__/test_trends.ambr | 204 ++-------- .../legacy_compatibility/feature_flag.py | 21 + 10 files changed, 198 insertions(+), 556 deletions(-) diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 754d4af18f97e..c599a8f2f8da6 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -206,6 +206,7 @@ export const FEATURE_FLAGS = { WEB_ANALYTICS_LIVE_USER_COUNT: 'web-analytics-live-user-count', // owner: @robbie-c SETTINGS_SESSION_TABLE_VERSION: 'settings-session-table-version', // owner: @robbie-c INSIGHT_FUNNELS_USE_UDF: 'insight-funnels-use-udf', // owner: @aspicer #team-product-analytics + INSIGHT_FUNNELS_USE_UDF_TRENDS: 'insight-funnels-use-udf-trends', // owner: @aspicer #team-product-analytics FIRST_TIME_FOR_USER_MATH: 'first-time-for-user-math', // owner: @skoob13 #team-product-analytics MULTITAB_EDITOR: 'multitab-editor', // owner: @EDsCODE #team-data-warehouse WEB_ANALYTICS_REPLAY: 'web-analytics-replay', // owner: @robbie-c diff --git a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py index ca10680f9f6a1..dcd5056d70bfa 100644 --- a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py +++ b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py @@ -1,4 +1,5 @@ from datetime import timedelta + from posthog.hogql.constants import HogQLGlobalSettings, MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY from math import ceil from typing import Optional, Any @@ -19,7 +20,10 @@ from posthog.hogql_queries.insights.funnels.funnel_trends import FunnelTrends from posthog.hogql_queries.insights.funnels.funnel_trends_udf import FunnelTrendsUDF from posthog.hogql_queries.insights.funnels.utils import get_funnel_actor_class, get_funnel_order_class -from posthog.hogql_queries.legacy_compatibility.feature_flag import insight_funnels_use_udf +from posthog.hogql_queries.legacy_compatibility.feature_flag import ( + insight_funnels_use_udf, + insight_funnels_use_udf_trends, +) from posthog.hogql_queries.query_runner import QueryRunner from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.models import Team @@ -111,7 +115,14 @@ def calculate(self): @cached_property def _use_udf(self): - return self.context.funnelsFilter.useUdf or insight_funnels_use_udf(self.team) + if self.context.funnelsFilter.useUdf: + return True + funnelVizType = self.context.funnelsFilter.funnelVizType + if funnelVizType == FunnelVizType.TRENDS and insight_funnels_use_udf_trends(self.team): + return True + if funnelVizType == FunnelVizType.STEPS and insight_funnels_use_udf(self.team): + return True + return False @cached_property def funnel_order_class(self): diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr index 0c52cf349a36b..0bf830178911f 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr @@ -1,366 +1,4 @@ # serializer version: 1 -# name: BaseTestFunnelTrends.test_timezones_trends - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'UTC'))), toIntervalDay(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'UTC')))), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfDay(timestamp) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e__pdi.person_id AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-04-30 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_timezones_trends.1 - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'US/Pacific'))), toIntervalDay(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'US/Pacific'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'US/Pacific')))), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfDay(timestamp) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, - e__pdi.person_id AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2021-04-30 00:00:00.000000', 6, 'US/Pacific')), lessOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'US/Pacific'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_week_interval - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-01 00:00:00'), 6, 'UTC')), 0), toIntervalWeek(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('week', toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-01 00:00:00'), 6, 'UTC')), 0), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'UTC')), 0)), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfWeek(timestamp, 0) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e__pdi.person_id AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_week_interval.1 - ''' - - SELECT aggregation_target AS actor_id - FROM - (SELECT aggregation_target, - toStartOfWeek(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC'), 0) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT *, - if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY - AND latest_1 <= latest_2 - AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps , - if(isNotNull(latest_1) - AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, - if(isNotNull(latest_2) - AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - if(latest_2 < latest_1, NULL, latest_2) as latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - min(latest_1) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT e.timestamp as timestamp, - pdi.person_id as aggregation_target, - pdi.person_id as person_id, - if(event = 'step one', 1, 0) as step_0, - if(step_0 = 1, timestamp, null) as latest_0, - if(event = 'step two', 1, 0) as step_1, - if(step_1 = 1, timestamp, null) as latest_1, - if(event = 'step three', 1, 0) as step_2, - if(step_2 = 1, timestamp, null) as latest_2 - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - AND distinct_id IN - (SELECT distinct_id - FROM events - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-05-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-05-07 23:59:59', 'UTC') ) - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-05-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-05-07 23:59:59', 'UTC') - AND (step_0 = 1 - OR step_1 = 1 - OR step_2 = 1) )))) - WHERE step_0 = 1 ) - WHERE toDateTime(entrance_period_start) = '2021-04-25 00:00:00' - GROUP BY aggregation_target, - entrance_period_start) - WHERE steps_completed >= 3 - ORDER BY aggregation_target - LIMIT 100 - OFFSET 0 SETTINGS max_ast_elements=1000000, - max_expanded_ast_elements=1000000 - ''' -# --- # name: TestFunnelTrendsUDF.test_timezones_trends ''' SELECT fill.entrance_period_start AS entrance_period_start, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr index e8487d4a42a9e..b02a7aa6e114c 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr @@ -355,7 +355,7 @@ WHERE equals(person_distinct_id2.team_id, 2) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - INNER JOIN + LEFT JOIN (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age @@ -463,7 +463,7 @@ WHERE equals(person_distinct_id2.team_id, 2) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - INNER JOIN + LEFT JOIN (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age @@ -582,7 +582,7 @@ WHERE equals(person_distinct_id2.team_id, 2) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - INNER JOIN + LEFT JOIN (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age @@ -701,7 +701,7 @@ WHERE equals(person_distinct_id2.team_id, 2) GROUP BY person_distinct_id2.distinct_id HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - INNER JOIN + LEFT JOIN (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py index 55a1f8660d0ca..446a529113f9f 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py @@ -11,7 +11,7 @@ from posthog.models.cohort.cohort import Cohort from posthog.models.filters import Filter from posthog.queries.funnels.funnel_trends_persons import ClickhouseFunnelTrendsActors -from posthog.schema import FunnelsQuery +from posthog.schema import FunnelsQuery, FunnelsQueryResponse from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, @@ -25,6 +25,7 @@ class BaseTestFunnelTrends(ClickhouseTestMixin, APIBaseTest): + __test__ = False maxDiff = None def _get_actors_at_step(self, filter, entrance_period_start, drop_off): @@ -1616,4 +1617,24 @@ def test_parses_breakdown_correctly(self): @patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestFunnelTrends(BaseTestFunnelTrends): - pass + __test__ = True + + def test_assert_flag_is_working(self): + filters = { + "insight": INSIGHT_FUNNELS, + "funnel_viz_type": "trends", + "display": TRENDS_LINEAR, + "interval": "hour", + "date_from": "2021-05-01 00:00:00", + "funnel_window_interval": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + + query = cast(FunnelsQuery, filter_to_query(filters)) + results = cast(FunnelsQueryResponse, FunnelsQueryRunner(query=query, team=self.team).calculate()) + + self.assertFalse(results.isUdf) diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py index 6965222b749f5..0b28f49fb952c 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py @@ -1,8 +1,55 @@ +from typing import cast from unittest.mock import patch, Mock +from posthog.constants import INSIGHT_FUNNELS, TRENDS_LINEAR +from posthog.hogql_queries.insights.funnels.funnels_query_runner import FunnelsQueryRunner from posthog.hogql_queries.insights.funnels.test.test_funnel_trends import BaseTestFunnelTrends +from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query +from posthog.schema import FunnelsQuery, FunnelsQueryResponse -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) +@patch( + "posthoganalytics.feature_enabled", + new=Mock(side_effect=lambda key, *args, **kwargs: key == "insight-funnels-use-udf-trends"), +) class TestFunnelTrendsUDF(BaseTestFunnelTrends): - pass + __test__ = True + + def test_assert_trends_flag_is_on(self): + filters = { + "insight": INSIGHT_FUNNELS, + "funnel_viz_type": "trends", + "display": TRENDS_LINEAR, + "interval": "hour", + "date_from": "2021-05-01 00:00:00", + "funnel_window_interval": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + + query = cast(FunnelsQuery, filter_to_query(filters)) + results = cast(FunnelsQueryResponse, FunnelsQueryRunner(query=query, team=self.team).calculate()) + + self.assertTrue(results.isUdf) + + def test_assert_steps_flag_is_off(self): + filters = { + "insight": INSIGHT_FUNNELS, + "funnel_viz_type": "steps", + "interval": "hour", + "date_from": "2021-05-01 00:00:00", + "funnel_window_interval": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + + query = cast(FunnelsQuery, filter_to_query(filters)) + results = cast(FunnelsQueryResponse, FunnelsQueryRunner(query=query, team=self.team).calculate()) + + self.assertFalse(results.isUdf) diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py index 2844d4b7792d4..849760ed262a3 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py @@ -1,13 +1,17 @@ +from typing import cast from unittest.mock import patch, Mock -from posthog.constants import FunnelOrderType +from posthog.constants import FunnelOrderType, INSIGHT_FUNNELS from posthog.hogql_queries.insights.funnels import Funnel +from posthog.hogql_queries.insights.funnels.funnels_query_runner import FunnelsQueryRunner from posthog.hogql_queries.insights.funnels.test.breakdown_cases import ( funnel_breakdown_test_factory, funnel_breakdown_group_test_factory, ) +from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query from posthog.models import Action from posthog.queries.funnels import ClickhouseFunnelActors +from posthog.schema import FunnelsQuery, FunnelsQueryResponse from posthog.test.base import ( ClickhouseTestMixin, _create_event, @@ -27,7 +31,10 @@ def _create_action(**kwargs): return action -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) +funnel_flag_side_effect = lambda key, *args, **kwargs: key == "insight-funnels-use-udf" + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) class TestFunnelBreakdownUDF( ClickhouseTestMixin, funnel_breakdown_test_factory( # type: ignore @@ -41,7 +48,7 @@ class TestFunnelBreakdownUDF( pass -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) class TestFunnelGroupBreakdownUDF( ClickhouseTestMixin, funnel_breakdown_group_test_factory( # type: ignore @@ -52,12 +59,50 @@ class TestFunnelGroupBreakdownUDF( pass -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) class TestFOSSFunnelUDF(funnel_test_factory(Funnel, _create_event, _create_person)): # type: ignore + def test_assert_flag_is_on(self): + filters = { + "insight": INSIGHT_FUNNELS, + "funnel_viz_type": "steps", + "interval": "hour", + "date_from": "2021-05-01 00:00:00", + "funnel_window_interval": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + + query = cast(FunnelsQuery, filter_to_query(filters)) + results = cast(FunnelsQueryResponse, FunnelsQueryRunner(query=query, team=self.team).calculate()) + + self.assertTrue(results.isUdf) + + def test_assert_trends_flag_is_off(self): + filters = { + "insight": INSIGHT_FUNNELS, + "funnel_viz_type": "trends", + "interval": "hour", + "date_from": "2021-05-01 00:00:00", + "funnel_window_interval": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + + query = cast(FunnelsQuery, filter_to_query(filters)) + results = cast(FunnelsQueryResponse, FunnelsQueryRunner(query=query, team=self.team).calculate()) + + self.assertFalse(results.isUdf) + maxDiff = None -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) class TestFunnelConversionTimeUDF( ClickhouseTestMixin, funnel_conversion_time_test_factory(FunnelOrderType.ORDERED, ClickhouseFunnelActors), # type: ignore diff --git a/posthog/hogql_queries/insights/funnels/utils.py b/posthog/hogql_queries/insights/funnels/utils.py index 31c31fa9b361d..ba8425843e5ed 100644 --- a/posthog/hogql_queries/insights/funnels/utils.py +++ b/posthog/hogql_queries/insights/funnels/utils.py @@ -32,13 +32,11 @@ def get_funnel_actor_class(funnelsFilter: FunnelsFilter): if funnelsFilter.funnelVizType == FunnelVizType.TRENDS: return FunnelTrendsActors - else: - if funnelsFilter.funnelOrderType == StepOrderValue.UNORDERED: - return FunnelUnorderedActors - elif funnelsFilter.funnelOrderType == StepOrderValue.STRICT: - return FunnelStrictActors - else: - return FunnelActors + if funnelsFilter.funnelOrderType == StepOrderValue.UNORDERED: + return FunnelUnorderedActors + if funnelsFilter.funnelOrderType == StepOrderValue.STRICT: + return FunnelStrictActors + return FunnelActors def funnel_window_interval_unit_to_sql( diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 2fe83c79dc067..0f3ec9882b914 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -853,49 +853,14 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.1 ''' - SELECT groupArray(1)(date)[1] AS date, - arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, - if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value - FROM - (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - breakdown_value AS breakdown_value, - rowNumberInAllBlocks() AS row_number - FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value - FROM - (SELECT count(DISTINCT e__pdi.person_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value - FROM events AS e SAMPLE 1.0 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) - GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) - WHERE isNotNull(breakdown_value) - GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC - LIMIT 50000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0 + /* celery:posthog.tasks.tasks.sync_insight_caching_state */ + SELECT team_id, + date_diff('second', max(timestamp), now()) AS age + FROM events + WHERE timestamp > date_sub(DAY, 3, now()) + AND timestamp < now() + GROUP BY team_id + ORDER BY age; ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.10 @@ -1112,143 +1077,38 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2 ''' - SELECT groupArray(1)(date)[1] AS date, - arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, - if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value - FROM - (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - breakdown_value AS breakdown_value, - rowNumberInAllBlocks() AS row_number - FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value - FROM - (SELECT count(DISTINCT e__pdi.person_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value - FROM events AS e SAMPLE 1.0 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) - GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) - WHERE isNotNull(breakdown_value) - GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC - LIMIT 50000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0 + /* celery:posthog.tasks.tasks.sync_insight_caching_state */ + SELECT team_id, + date_diff('second', max(timestamp), now()) AS age + FROM events + WHERE timestamp > date_sub(DAY, 3, now()) + AND timestamp < now() + GROUP BY team_id + ORDER BY age; ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.3 ''' - SELECT groupArray(1)(date)[1] AS date, - arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, - arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value - FROM - (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - breakdown_value AS breakdown_value, - rowNumberInAllBlocks() AS row_number - FROM - (SELECT sum(total) AS count, - day_start AS day_start, - [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value - FROM - (SELECT count(DISTINCT e__pdi.person_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 - FROM events AS e SAMPLE 1.0 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY day_start, - breakdown_value_1) - GROUP BY day_start, - breakdown_value_1 - ORDER BY day_start ASC, breakdown_value ASC) - GROUP BY breakdown_value - ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) - WHERE arrayExists(x -> isNotNull(x), breakdown_value) - GROUP BY breakdown_value - ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC - LIMIT 50000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0 + /* celery:posthog.tasks.tasks.sync_insight_caching_state */ + SELECT team_id, + date_diff('second', max(timestamp), now()) AS age + FROM events + WHERE timestamp > date_sub(DAY, 3, now()) + AND timestamp < now() + GROUP BY team_id + ORDER BY age; ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.4 ''' - SELECT groupArray(1)(date)[1] AS date, - arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, - arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value - FROM - (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - breakdown_value AS breakdown_value, - rowNumberInAllBlocks() AS row_number - FROM - (SELECT sum(total) AS count, - day_start AS day_start, - [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value - FROM - (SELECT count(DISTINCT e__pdi.person_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 - FROM events AS e SAMPLE 1.0 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY day_start, - breakdown_value_1) - GROUP BY day_start, - breakdown_value_1 - ORDER BY day_start ASC, breakdown_value ASC) - GROUP BY breakdown_value - ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) - WHERE arrayExists(x -> isNotNull(x), breakdown_value) - GROUP BY breakdown_value - ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC - LIMIT 50000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0 + /* celery:posthog.tasks.tasks.sync_insight_caching_state */ + SELECT team_id, + date_diff('second', max(timestamp), now()) AS age + FROM events + WHERE timestamp > date_sub(DAY, 3, now()) + AND timestamp < now() + GROUP BY team_id + ORDER BY age; ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.5 diff --git a/posthog/hogql_queries/legacy_compatibility/feature_flag.py b/posthog/hogql_queries/legacy_compatibility/feature_flag.py index 697e479f6031f..ff9eaa105d047 100644 --- a/posthog/hogql_queries/legacy_compatibility/feature_flag.py +++ b/posthog/hogql_queries/legacy_compatibility/feature_flag.py @@ -42,3 +42,24 @@ def insight_funnels_use_udf(team: Team) -> bool: only_evaluate_locally=False, send_feature_flag_events=False, ) + + +def insight_funnels_use_udf_trends(team: Team) -> bool: + return posthoganalytics.feature_enabled( + "insight-funnels-use-udf-trends", + str(team.uuid), + groups={ + "organization": str(team.organization_id), + "project": str(team.id), + }, + group_properties={ + "organization": { + "id": str(team.organization_id), + }, + "project": { + "id": str(team.id), + }, + }, + only_evaluate_locally=False, + send_feature_flag_events=False, + )