From 4f305131eed223e0a401620c173f778fec61e16b Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Mon, 19 Feb 2024 11:28:41 -0800 Subject: [PATCH] Brevo: add batch send support Closes #353 --- CHANGELOG.rst | 12 +++ anymail/backends/sendinblue.py | 66 ++++++++++++--- docs/esps/brevo.rst | 84 ++++++++++++++---- tests/test_sendinblue_backend.py | 122 +++++++++++++++++++++------ tests/test_sendinblue_integration.py | 52 ++++++------ 5 files changed, 258 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1123974d..9ed5507b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,18 @@ Release history ^^^^^^^^^^^^^^^ .. This extra heading level keeps the ToC from becoming unmanageably long +vNext +----- + +*unreleased changes* + +Features +~~~~~~~~ + +* **Brevo:** Add support for batch sending + (`docs `__). + + v10.2 ----- diff --git a/anymail/backends/sendinblue.py b/anymail/backends/sendinblue.py index caf1b9f6..e469c915 100644 --- a/anymail/backends/sendinblue.py +++ b/anymail/backends/sendinblue.py @@ -39,27 +39,41 @@ def parse_recipient_status(self, response, payload, message): # SendinBlue doesn't give any detail on a success # https://developers.sendinblue.com/docs/responses message_id = None + message_ids = [] if response.content != b"": parsed_response = self.deserialize_json_response(response, payload, message) try: message_id = parsed_response["messageId"] - except (KeyError, TypeError) as err: - raise AnymailRequestsAPIError( - "Invalid SendinBlue API response format", - email_message=message, - payload=payload, - response=response, - backend=self, - ) from err + except (KeyError, TypeError): + try: + # batch send + message_ids = parsed_response["messageIds"] + except (KeyError, TypeError) as err: + raise AnymailRequestsAPIError( + "Invalid SendinBlue API response format", + email_message=message, + payload=payload, + response=response, + backend=self, + ) from err status = AnymailRecipientStatus(message_id=message_id, status="queued") - return {recipient.addr_spec: status for recipient in payload.all_recipients} + recipient_status = { + recipient.addr_spec: status for recipient in payload.all_recipients + } + if message_ids: + for to, message_id in zip(payload.to_recipients, message_ids): + recipient_status[to.addr_spec] = AnymailRecipientStatus( + message_id=message_id, status="queued" + ) + return recipient_status class SendinBluePayload(RequestsPayload): def __init__(self, message, defaults, backend, *args, **kwargs): self.all_recipients = [] # used for backend.parse_recipient_status + self.to_recipients = [] # used for backend.parse_recipient_status http_headers = kwargs.pop("headers", {}) http_headers["api-key"] = backend.api_key @@ -74,9 +88,32 @@ def get_api_endpoint(self): def init_payload(self): self.data = {"headers": CaseInsensitiveDict()} # becomes json + self.merge_data = {} + self.metadata = {} + self.merge_metadata = {} def serialize_data(self): """Performs any necessary serialization on self.data, and returns the result.""" + if self.is_batch(): + # Burst data["to"] into data["messageVersions"] + to_list = self.data.pop("to", []) + self.data["messageVersions"] = [ + {"to": [to], "params": self.merge_data.get(to["email"])} + for to in to_list + ] + if self.merge_metadata: + # Merge global metadata with any per-recipient metadata. + # (Top-level X-Mailin-custom header is already set to global metadata, + # and will apply for recipients without a "headers" override.) + for version in self.data["messageVersions"]: + to_email = version["to"][0]["email"] + if to_email in self.merge_metadata: + recipient_metadata = self.metadata.copy() + recipient_metadata.update(self.merge_metadata[to_email]) + version["headers"] = { + "X-Mailin-custom": self.serialize_json(recipient_metadata) + } + if not self.data["headers"]: del self.data["headers"] # don't send empty headers return self.serialize_json(self.data) @@ -102,6 +139,8 @@ def set_recipients(self, recipient_type, emails): if emails: self.data[recipient_type] = [self.email_object(email) for email in emails] self.all_recipients += emails # used for backend.parse_recipient_status + if recipient_type == "to": + self.to_recipients = emails # used for backend.parse_recipient_status def set_subject(self, subject): if subject != "": # see note in set_text_body about template rendering @@ -158,8 +197,8 @@ def set_esp_extra(self, extra): self.data.update(extra) def set_merge_data(self, merge_data): - """SendinBlue doesn't support special attributes for each recipient""" - self.unsupported_feature("merge_data") + # Late bound in serialize_data: + self.merge_data = merge_data def set_merge_global_data(self, merge_global_data): self.data["params"] = merge_global_data @@ -167,6 +206,11 @@ def set_merge_global_data(self, merge_global_data): def set_metadata(self, metadata): # SendinBlue expects a single string payload self.data["headers"]["X-Mailin-custom"] = self.serialize_json(metadata) + self.metadata = metadata # needed in serialize_data for batch send + + def set_merge_metadata(self, merge_metadata): + # Late-bound in serialize_data: + self.merge_metadata = merge_metadata def set_send_at(self, send_at): try: diff --git a/docs/esps/brevo.rst b/docs/esps/brevo.rst index 6934c251..acba2773 100644 --- a/docs/esps/brevo.rst +++ b/docs/esps/brevo.rst @@ -149,10 +149,33 @@ Brevo can handle. If you are ignoring unsupported features and have multiple reply addresses, Anymail will use only the first one. -**Metadata** +**Metadata exposed in message headers** Anymail passes :attr:`~anymail.message.AnymailMessage.metadata` to Brevo as a JSON-encoded string using their :mailheader:`X-Mailin-custom` email header. - The metadata is available in tracking webhooks. + This header is included in the sent message, so **metadata will be visible to + message recipients** if they view the raw message source. + +**Special headers** + Brevo uses special email headers to control certain features. + You can set these using Django's + :class:`EmailMessage.headers `: + + .. code-block:: python + + message = EmailMessage( + ..., + headers = { + "sender.ip": "10.10.1.150", # use a dedicated IP + "idempotencyKey": "...uuid...", # batch send deduplication + } + ) + + # Note the constructor param is called `headers`, but the + # corresponding attribute is named `extra_headers`: + message.extra_headers = { + "sender.ip": "10.10.1.222", + "idempotencyKey": "...uuid...", + } **Delayed sending** .. versionadded:: 9.0 @@ -174,30 +197,33 @@ Brevo can handle. Batch sending/merge and ESP templates ------------------------------------- -Brevo supports :ref:`ESP stored templates ` populated with -global merge data for all recipients, but does not offer :ref:`batch sending ` -with per-recipient merge data. Anymail's :attr:`~anymail.message.AnymailMessage.merge_data` -and :attr:`~anymail.message.AnymailMessage.merge_metadata` message attributes are not -supported with the Brevo backend, but you can use Anymail's -:attr:`~anymail.message.AnymailMessage.merge_global_data` with Brevo templates. +.. versionchanged:: 10.3 + + Added support for batch sending with :attr:`~anymail.message.AnymailMessage.merge_data` + and :attr:`~anymail.message.AnymailMessage.merge_metadata`. + +Brevo supports :ref:`ESP stored templates ` and +:ref:`batch sending ` with per-recipient merge data. To use a Brevo template, set the message's :attr:`~anymail.message.AnymailMessage.template_id` to the numeric -Brevo template ID, and supply substitution attributes using -the message's :attr:`~anymail.message.AnymailMessage.merge_global_data`: +Brevo template ID, and supply substitution params using Anymail's normalized +:attr:`~anymail.message.AnymailMessage.merge_data` and +:attr:`~anymail.message.AnymailMessage.merge_global_data` message attributes: .. code-block:: python message = EmailMessage( - to=["alice@example.com"] # single recipient... - # ...multiple to emails would all get the same message - # (and would all see each other's emails in the "to" header) + # (subject and body come from the template, so don't include those) + to=["alice@example.com", "Bob "] ) message.template_id = 3 # use this Brevo template message.from_email = None # to use the template's default sender + message.merge_data = { + 'alice@example.com': {'name': "Alice", 'order_no': "12345"}, + 'bob@example.com': {'name': "Bob", 'order_no': "54321"}, + } message.merge_global_data = { - 'name': "Alice", - 'order_no': "12345", 'ship_date': "May 15", } @@ -214,6 +240,31 @@ If you want to use the template's sender, be sure to set ``from_email`` to ``Non You can also override the template's subject and reply-to address (but not body) using standard :class:`~django.core.mail.EmailMessage` attributes. +Brevo also supports batch-sending without using an ESP-stored template. In this +case, each recipient will receive the same content (Brevo doesn't support inline +templates) but will see only their own *To* email address. Setting either of +:attr:`~anymail.message.AnymailMessage.merge_data` or +:attr:`~anymail.message.AnymailMessage.merge_metadata`---even to an empty +dict---will cause Anymail to use Brevo's batch send option (``"messageVersions"``). + +You can use Anymail's +:attr:`~anymail.message.AnymailMessage.merge_metadata` to supply custom tracking +data for each recipient: + + .. code-block:: python + + message = EmailMessage( + to=["alice@example.com", "Bob "], + from_email="...", subject="...", body="..." + ) + message.merge_metadata = { + 'alice@example.com': {'user_id': "12345"}, + 'bob@example.com': {'user_id': "54321"}, + } + +To use Brevo's "`idempotencyKey`_" with a batch send, set it in the +message's headers: ``message.extra_headers = {"idempotencyKey": "...uuid..."}``. + .. caution:: **Sendinblue "old template language" not supported** @@ -241,6 +292,9 @@ using standard :class:`~django.core.mail.EmailMessage` attributes. .. _Brevo Template Language: https://help.brevo.com/hc/en-us/articles/360000946299 +.. _idempotencyKey: + https://developers.brevo.com/docs/heterogenous-versions-batch-emails + .. _convert each old template: https://help.brevo.com/hc/en-us/articles/360000991960 diff --git a/tests/test_sendinblue_backend.py b/tests/test_sendinblue_backend.py index f22e2d5b..d1290c78 100644 --- a/tests/test_sendinblue_backend.py +++ b/tests/test_sendinblue_backend.py @@ -18,7 +18,7 @@ AnymailSerializationError, AnymailUnsupportedFeature, ) -from anymail.message import attach_inline_image_file +from anymail.message import AnymailMessage, attach_inline_image_file from .mock_requests_backend import ( RequestsBackendMockAPITestCase, @@ -478,13 +478,60 @@ def test_template_id(self): self.assertEqual(data["subject"], "My Subject") self.assertEqual(data["to"], [{"email": "to@example.com", "name": "Recipient"}]) + _mock_batch_response = { + "messageIds": [ + "<202403182259.64789700810.1@smtp-relay.mailin.fr>", + "<202403182259.64789700810.2@smtp-relay.mailin.fr>", + ] + } + def test_merge_data(self): - self.message.merge_data = { - "alice@example.com": {":name": "Alice", ":group": "Developers"}, - "bob@example.com": {":name": "Bob"}, # and leave :group undefined - } - with self.assertRaises(AnymailUnsupportedFeature): - self.message.send() + self.set_mock_response(json_data=self._mock_batch_response) + message = AnymailMessage( + from_email="from@example.com", + template_id=1234567, + to=["alice@example.com", "Bob "], + merge_data={ + "alice@example.com": {"name": "Alice", "group": "Developers"}, + "bob@example.com": {"name": "Bob"}, # and leave group undefined + "nobody@example.com": {"name": "Not a recipient for this message"}, + }, + merge_global_data={"group": "Users", "site": "ExampleCo"}, + ) + message.send() + + # batch send uses same API endpoint as regular send: + self.assert_esp_called("/v3/smtp/email") + data = self.get_api_call_json() + versions = data["messageVersions"] + self.assertEqual(len(versions), 2) + self.assertEqual( + versions[0], + { + "to": [{"email": "alice@example.com"}], + "params": {"name": "Alice", "group": "Developers"}, + }, + ) + self.assertEqual( + versions[1], + { + "to": [{"email": "bob@example.com", "name": "Bob"}], + "params": {"name": "Bob"}, + }, + ) + self.assertEqual(data["params"], {"group": "Users", "site": "ExampleCo"}) + + recipients = message.anymail_status.recipients + self.assertEqual(recipients["alice@example.com"].status, "queued") + self.assertEqual( + recipients["alice@example.com"].message_id, + "<202403182259.64789700810.1@smtp-relay.mailin.fr>", + ) + self.assertEqual(recipients["bob@example.com"].status, "queued") + self.assertEqual( + recipients["bob@example.com"].message_id, + "<202403182259.64789700810.2@smtp-relay.mailin.fr>", + ) def test_merge_global_data(self): self.message.merge_global_data = {"a": "b"} @@ -492,6 +539,38 @@ def test_merge_global_data(self): data = self.get_api_call_json() self.assertEqual(data["params"], {"a": "b"}) + def test_merge_metadata(self): + self.set_mock_response(json_data=self._mock_batch_response) + self.message.to = ["alice@example.com", "Bob "] + self.message.merge_metadata = { + "alice@example.com": {"order_id": 123, "tier": "premium"}, + "bob@example.com": {"order_id": 678}, + } + self.message.metadata = {"notification_batch": "zx912"} + self.message.send() + + data = self.get_api_call_json() + versions = data["messageVersions"] + self.assertEqual(len(versions), 2) + self.assertEqual(versions[0]["to"], [{"email": "alice@example.com"}]) + # metadata and merge_metadata[recipient] are combined: + self.assertEqual( + json.loads(versions[0]["headers"]["X-Mailin-custom"]), + {"order_id": 123, "tier": "premium", "notification_batch": "zx912"}, + ) + self.assertEqual( + versions[1]["to"], [{"name": "Bob", "email": "bob@example.com"}] + ) + self.assertEqual( + json.loads(versions[1]["headers"]["X-Mailin-custom"]), + {"order_id": 678, "notification_batch": "zx912"}, + ) + # default metadata still sent in base headers: + self.assertEqual( + json.loads(data["headers"]["X-Mailin-custom"]), + {"notification_batch": "zx912"}, + ) + def test_default_omits_options(self): """Make sure by default we don't send any ESP-specific options. @@ -502,35 +581,24 @@ def test_default_omits_options(self): self.message.send() data = self.get_api_call_json() self.assertNotIn("attachment", data) - self.assertNotIn("tag", data) + self.assertNotIn("bcc", data) + self.assertNotIn("cc", data) self.assertNotIn("headers", data) + self.assertNotIn("messageVersions", data) + self.assertNotIn("params", data) self.assertNotIn("replyTo", data) - self.assertNotIn("atributes", data) + self.assertNotIn("schedule", data) + self.assertNotIn("tags", data) + self.assertNotIn("templateId", data) def test_esp_extra(self): - # SendinBlue doesn't offer any esp-extra but we will test - # with some extra of SendGrid to see if it's work in the future self.message.esp_extra = { - "ip_pool_name": "transactional", - "asm": { # subscription management - "group_id": 1, - }, - "tracking_settings": { - "subscription_tracking": { - "enable": True, - "substitution_tag": "[unsubscribe_url]", - }, - }, + "batchId": "5c6cfa04-eed9-42c2-8b5c-6d470d978e9d", } self.message.send() data = self.get_api_call_json() # merged from esp_extra: - self.assertEqual(data["ip_pool_name"], "transactional") - self.assertEqual(data["asm"], {"group_id": 1}) - self.assertEqual( - data["tracking_settings"]["subscription_tracking"], - {"enable": True, "substitution_tag": "[unsubscribe_url]"}, - ) + self.assertEqual(data["batchId"], "5c6cfa04-eed9-42c2-8b5c-6d470d978e9d") # noinspection PyUnresolvedReferences def test_send_attaches_anymail_status(self): diff --git a/tests/test_sendinblue_integration.py b/tests/test_sendinblue_integration.py index 3d235965..0acdb2dd 100644 --- a/tests/test_sendinblue_integration.py +++ b/tests/test_sendinblue_integration.py @@ -98,40 +98,42 @@ def test_template(self): template_id=5, # Override template sender: from_email=formataddr(("Sender", self.from_email)), - # No batch send (so max one recipient suggested): - to=["Recipient "], + to=["Recipient 1 ", "test+to2@anymail.dev"], reply_to=["Do not reply "], tags=["using-template"], - headers={"X-Anymail-Test": "group: A, variation: C"}, - merge_global_data={ - # The Anymail test template includes `{{ params.SHIP_DATE }}` - # and `{{ params.ORDER_ID }}` substitutions - "SHIP_DATE": "yesterday", - "ORDER_ID": "12345", + # The Anymail test template includes `{{ params.SHIP_DATE }}` + # and `{{ params.ORDER_ID }}` substitutions + merge_data={ + "test+to1@anymail.dev": {"ORDER_ID": "12345"}, + "test+to2@anymail.dev": {"ORDER_ID": "23456"}, + }, + merge_global_data={"SHIP_DATE": "yesterday"}, + metadata={"customer-id": "unknown", "meta2": 2}, + merge_metadata={ + "test+to1@anymail.dev": {"customer-id": "ZXK9123"}, + "test+to2@anymail.dev": {"customer-id": "ZZT4192"}, }, - metadata={"customer-id": "ZXK9123", "meta2": 2}, ) - # Normal attachments don't work with Brevo templates: - # message.attach("attachment1.txt", "Here is some\ntext", "text/plain") - # If you can host the attachment content on some publicly-accessible URL, - # this *non-portable* alternative allows sending attachments with templates: - message.esp_extra = { - "attachment": [ - { - "name": "attachment1.txt", - # URL where Brevo can download the attachment content while - # sending (must be content-type: text/plain): - "url": "https://raw.githubusercontent.com/anymail/django-anymail/" - "main/docs/_readme/template.txt", - } - ] - } + message.attach("attachment1.txt", "Here is some\ntext", "text/plain") message.send() # SendinBlue always queues: self.assertEqual(message.anymail_status.status, {"queued"}) - self.assertRegex(message.anymail_status.message_id, r"\<.+@.+\>") + recipient_status = message.anymail_status.recipients + self.assertEqual(recipient_status["test+to1@anymail.dev"].status, "queued") + self.assertEqual(recipient_status["test+to2@anymail.dev"].status, "queued") + self.assertRegex( + recipient_status["test+to1@anymail.dev"].message_id, r"\<.+@.+\>" + ) + self.assertRegex( + recipient_status["test+to2@anymail.dev"].message_id, r"\<.+@.+\>" + ) + # Each recipient gets their own message_id: + self.assertNotEqual( + recipient_status["test+to1@anymail.dev"].message_id, + recipient_status["test+to2@anymail.dev"].message_id, + ) @override_settings(ANYMAIL_SENDINBLUE_API_KEY="Hey, that's not an API key!") def test_invalid_api_key(self):