feat(cdp): sendgrid template updates (#24814)

posthog/cdp/templates/__init__.py

@@ -3,7 +3,7 @@
 from .hubspot.template_hubspot import template as hubspot
 from .customerio.template_customerio import template as customerio, TemplateCustomerioMigrator
 from .intercom.template_intercom import template as intercom, TemplateIntercomMigrator
-from .sendgrid.template_sendgrid import template as sendgrid
+from .sendgrid.template_sendgrid import template as sendgrid, TemplateSendGridMigrator
 from .clearbit.template_clearbit import template as clearbit
 from .posthog.template_posthog import template as posthog
 from .aws_kinesis.template_aws_kinesis import template as aws_kinesis
@@ -44,6 +44,7 @@
 HOG_FUNCTION_MIGRATORS = {
     TemplateCustomerioMigrator.plugin_url: TemplateCustomerioMigrator,
     TemplateIntercomMigrator.plugin_url: TemplateIntercomMigrator,
+    TemplateSendGridMigrator.plugin_url: TemplateSendGridMigrator,
 }
 
 __all__ = ["HOG_FUNCTION_TEMPLATES", "HOG_FUNCTION_TEMPLATES_BY_ID"]

posthog/cdp/templates/sendgrid/template_sendgrid.py

@@ -1,4 +1,7 @@
-from posthog.cdp.templates.hog_function_template import HogFunctionTemplate
+import dataclasses
+from copy import deepcopy
+
+from posthog.cdp.templates.hog_function_template import HogFunctionTemplate, HogFunctionTemplateMigrator
 
 # Based off of https://www.twilio.com/docs/sendgrid/api-reference/contacts/add-or-update-a-contact
 
@@ -9,15 +12,13 @@
     description="Update marketing contacts in Sendgrid",
     icon_url="/static/services/sendgrid.png",
     hog="""
-let email := inputs.email
-
-if (empty(email)) {
+if (empty(inputs.email)) {
     print('`email` input is empty. Not updating contacts.')
     return
 }
 
 let contact := {
-  'email': email,
+  'email': inputs.email,
 }
 
 for (let key, value in inputs.properties) {
@@ -26,15 +27,32 @@
     }
 }
 
+let headers :=  {
+    'Authorization': f'Bearer {inputs.api_key}',
+    'Content-Type': 'application/json'
+}
+
+if (not empty(inputs.custom_fields)) {
+    let response := fetch('https://api.sendgrid.com/v3/marketing/field_definitions', {
+        'method': 'GET',
+        'headers': headers
+    })
+    if (response.status != 200) {
+        throw Error(f'Could not fetch custom fields. Status: {response.status}')
+    }
+    contact['custom_fields'] := {}
+    for (let obj in response.body?.custom_fields ?? {}) {
+        let inputValue := inputs.custom_fields[obj.name]
+        if (not empty(inputValue)) {
+            contact['custom_fields'][obj.id] := inputValue
+        }
+    }
+}
+
 let res := fetch('https://api.sendgrid.com/v3/marketing/contacts', {
     'method': 'PUT',
-    'headers': {
-        'Authorization': f'Bearer {inputs.api_key}',
-        'Content-Type': 'application/json'
-    },
-    'body': {
-      'contacts': [contact]
-    }
+    'headers': headers,
+    'body': { 'contacts': [contact] }
 })
 
 if (res.status > 300) {
@@ -61,23 +79,83 @@
         {
             "key": "properties",
             "type": "dictionary",
-            "label": "Property mapping",
-            "description": "Map of reserved properties (https://www.twilio.com/docs/sendgrid/api-reference/contacts/add-or-update-a-contact)",
+            "label": "Reserved fields",
+            "description": "The following field names are allowed: address_line_1, address_line_2, alternate_emails, anonymous_id, city, country, email, external_id, facebook, first_name, last_name, phone_number_id, postal_code, state_province_region, unique_name, whatsapp.",
             "default": {
-                "last_name": "{person.properties.last_name}",
                 "first_name": "{person.properties.first_name}",
+                "last_name": "{person.properties.last_name}",
                 "city": "{person.properties.city}",
                 "country": "{person.properties.country}",
                 "postal_code": "{person.properties.postal_code}",
             },
             "secret": False,
             "required": True,
         },
-        # TODO: Add dynamic code for loading custom fields
+        {
+            "key": "custom_fields",
+            "type": "dictionary",
+            "label": "Custom fields",
+            "description": "Configure custom fields in SendGrid before using them here: https://mc.sendgrid.com/custom-fields",
+            "default": {},
+            "secret": False,
+            "required": False,
+        },
     ],
     filters={
         "events": [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}],
         "actions": [],
         "filter_test_accounts": True,
     },
 )
+
+
+class TemplateSendGridMigrator(HogFunctionTemplateMigrator):
+    plugin_url = "https://github.com/PostHog/sendgrid-plugin"
+
+    @classmethod
+    def migrate(cls, obj):
+        hf = deepcopy(dataclasses.asdict(template))
+
+        sendgridApiKey = obj.config.get("sendgridApiKey", "")
+        customFields = obj.config.get("customFields", "")
+        sendgrid_fields = [
+            "address_line_1",
+            "address_line_2",
+            "alternate_emails",
+            "anonymous_id",
+            "city",
+            "country",
+            "email",
+            "external_id",
+            "facebook",
+            "first_name",
+            "last_name",
+            "phone_number_id",
+            "postal_code",
+            "state_province_region",
+            "unique_name",
+            "whatsapp",
+        ]
+
+        hf["filters"] = {}
+        hf["filters"]["events"] = [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}]
+
+        hf["inputs"] = {
+            "api_key": {"value": sendgridApiKey},
+            "email": {"value": "{person.properties.email}"},
+            "properties": {"value": {}},
+            "custom_fields": {"value": {}},
+        }
+        if customFields:
+            for field in customFields.split(","):
+                if "=" in field:
+                    posthog_prop, sendgrid_field = field.split("=")
+                else:
+                    posthog_prop = sendgrid_field = field.strip()
+                posthog_prop = f"{{person.properties.{posthog_prop}}}"
+                if sendgrid_field in sendgrid_fields:
+                    hf["inputs"]["properties"]["value"][sendgrid_field] = posthog_prop
+                else:
+                    hf["inputs"]["custom_fields"]["value"][sendgrid_field] = posthog_prop
+
+        return hf

posthog/cdp/templates/sendgrid/test_template_sendgrid.py

@@ -1,6 +1,8 @@
 from inline_snapshot import snapshot
 from posthog.cdp.templates.helpers import BaseHogFunctionTemplateTest
-from posthog.cdp.templates.sendgrid.template_sendgrid import template as template_sendgrid
+from posthog.cdp.templates.sendgrid.template_sendgrid import template as template_sendgrid, TemplateSendGridMigrator
+from posthog.models import PluginConfig
+from posthog.test.base import BaseTest
 
 
 class TestTemplateSendgrid(BaseHogFunctionTemplateTest):
@@ -47,3 +49,138 @@ def test_function_doesnt_include_empty_properties(self):
         assert self.get_mock_fetch_calls()[0][1]["body"]["contacts"] == snapshot(
             [{"email": "[email protected]", "last_name": "included"}]
         )
+
+    def test_function_adds_custom_fields(self):
+        self.mock_fetch_response = lambda *args: {  # type: ignore
+            "status": 200,
+            "body": {"custom_fields": [{"id": "id7", "name": "custom_field"}]},
+        }
+
+        res = self.run_function(
+            inputs=self._inputs(
+                custom_fields={"custom_field": "custom_value"},
+            )
+        )
+        assert res.result is None
+
+        assert self.get_mock_fetch_calls()[0] == snapshot(
+            (
+                "https://api.sendgrid.com/v3/marketing/field_definitions",
+                {
+                    "method": "GET",
+                    "headers": {"Authorization": "Bearer API_KEY", "Content-Type": "application/json"},
+                },
+            )
+        )
+
+        assert self.get_mock_fetch_calls()[1] == snapshot(
+            (
+                "https://api.sendgrid.com/v3/marketing/contacts",
+                {
+                    "method": "PUT",
+                    "headers": {"Authorization": "Bearer API_KEY", "Content-Type": "application/json"},
+                    "body": {
+                        "contacts": [
+                            {
+                                "email": "[email protected]",
+                                "last_name": "example",
+                                "custom_fields": {"id7": "custom_value"},
+                            }
+                        ]
+                    },
+                },
+            )
+        )
+
+
+class TestTemplateMigration(BaseTest):
+    def get_plugin_config(self, config: dict):
+        _config = {
+            "sendgridApiKey": "SENDGRID_API_KEY",
+            "customFields": "",
+        }
+        _config.update(config)
+        return PluginConfig(enabled=True, order=0, config=_config)
+
+    def test_empty_fields(self):
+        obj = self.get_plugin_config({})
+
+        template = TemplateSendGridMigrator.migrate(obj)
+        assert template["inputs"] == snapshot(
+            {
+                "api_key": {"value": "SENDGRID_API_KEY"},
+                "email": {"value": "{person.properties.email}"},
+                "custom_fields": {"value": {}},
+                "properties": {"value": {}},
+            }
+        )
+        assert template["filters"] == snapshot(
+            {"events": [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}]}
+        )
+
+    def test_default_properties(self):
+        obj = self.get_plugin_config({"customFields": "last_name,first_name"})
+
+        template = TemplateSendGridMigrator.migrate(obj)
+        assert template["inputs"] == snapshot(
+            {
+                "api_key": {"value": "SENDGRID_API_KEY"},
+                "email": {"value": "{person.properties.email}"},
+                "custom_fields": {"value": {}},
+                "properties": {
+                    "value": {
+                        "last_name": "{person.properties.last_name}",
+                        "first_name": "{person.properties.first_name}",
+                    }
+                },
+            }
+        )
+        assert template["filters"] == snapshot(
+            {"events": [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}]}
+        )
+
+    def test_custom_fields(self):
+        obj = self.get_plugin_config({"customFields": "last_name,first_name,misc_name,banana"})
+
+        template = TemplateSendGridMigrator.migrate(obj)
+        assert template["inputs"] == snapshot(
+            {
+                "api_key": {"value": "SENDGRID_API_KEY"},
+                "email": {"value": "{person.properties.email}"},
+                "custom_fields": {
+                    "value": {"misc_name": "{person.properties.misc_name}", "banana": "{person.properties.banana}"}
+                },
+                "properties": {
+                    "value": {
+                        "last_name": "{person.properties.last_name}",
+                        "first_name": "{person.properties.first_name}",
+                    }
+                },
+            }
+        )
+        assert template["filters"] == snapshot(
+            {"events": [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}]}
+        )
+
+    def test_property_rename(self):
+        obj = self.get_plugin_config({"customFields": "$lastName=last_name,first_name,misc_name,$pineapple=banana"})
+
+        template = TemplateSendGridMigrator.migrate(obj)
+        assert template["inputs"] == snapshot(
+            {
+                "api_key": {"value": "SENDGRID_API_KEY"},
+                "email": {"value": "{person.properties.email}"},
+                "custom_fields": {
+                    "value": {"misc_name": "{person.properties.misc_name}", "banana": "{person.properties.$pineapple}"}
+                },
+                "properties": {
+                    "value": {
+                        "last_name": "{person.properties.$lastName}",
+                        "first_name": "{person.properties.first_name}",
+                    }
+                },
+            }
+        )
+        assert template["filters"] == snapshot(
+            {"events": [{"id": "$identify", "name": "$identify", "type": "events", "order": 0}]}
+        )

posthog/hogql/bytecode.py

@@ -391,6 +391,14 @@ def visit_block(self, node: ast.Block):
     def visit_expr_statement(self, node: ast.ExprStatement):
         if node.expr is None:
             return []
+        if isinstance(node.expr, ast.CompareOperation) and node.expr.op == ast.CompareOperationOp.Eq:
+            self.context.warnings.append(
+                HogQLNotice(
+                    start=node.start,
+                    end=node.end,
+                    message="You must use ':=' for assignment instead of '='.",
+                )
+            )
         response = self.visit(node.expr)
         response.append(Operation.POP)
         return response