Merge pull request #1408 from elementary-data/ele-2501-fix-alert_v2-p…

…opulation-query-too-long ELE-2501 - Remove long columns from alerts_v2 population
elementary-data · Feb 13, 2024 · 5502f54 · 5502f54
2 parents 3504198 + e99227a
commit 5502f54
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 6 deletions.
diff --git a/elementary/monitor/dbt_project/macros/alerts/population/populate_alerts_table.sql b/elementary/monitor/dbt_project/macros/alerts/population/populate_alerts_table.sql
@@ -16,7 +16,7 @@
             {% endif %}
         {% endfor %}
 
-        {% do elementary.insert_rows(alerts_v2_relation, unhandled_alerts) %}
+        {% do elementary.insert_rows(alerts_v2_relation, unhandled_alerts, on_query_exceed=populate_alerts_on_query_exceed) %}
     {% endif %}
     {% do return('') %}
 {% endmacro %}
@@ -57,3 +57,20 @@
 
     {% do return(alert_ids) %}
 {% endmacro %}
+
+
+{% macro populate_alerts_on_query_exceed(alert_row) %}
+    {% set row_max_size = elementary.get_config_var('query_max_size') %}
+
+    {# alert data contains data that could exceed the query size limit #}
+    {# We remove the problematic fields to insure the query is in the right size #}
+    {% set alert_data = alert_row['data'] %}
+    {% set alert_data_dict = fromjson(alert_data) %}
+    {% set risky_fields = ['test_rows_sample', 'test_results_query'] %}
+    {% for risky_field in risky_fields %}
+        {% if (tojson(alert_data_dict[risky_field]) | length) > (row_max_size / 3) %}
+            {% do alert_data_dict.update({risky_field: none}) %}            
+        {% endif %}
+    {% endfor %}
+    {% do alert_row.update({'data': tojson(alert_data_dict)}) %}
+{% endmacro %}
diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql
@@ -42,11 +42,6 @@
         } 
         %}
 
-        {% set column_size = elementary.get_column_size() %}
-        {% if column_size and ((tojson(test_alert_data) | length) > column_size) %}
-            {% do test_alert_data.update({'test_rows_sample': none}) %}  
-        {% endif %}
-
         {% set test_alert = elementary_cli.generate_alert_object(
             raw_test_alert.alert_id,
             raw_test_alert.alert_class_id,