Identify scrapy-zyte-api usage via custom user-agent

.bumpversion.cfg

@@ -4,6 +4,4 @@ commit = True
 tag = True
 tag_name = {new_version}
 
-[bumpversion:file:setup.py]
-search = version="{current_version}",
-replace = version="{new_version}",
+[bumpversion:file:scrapy_zyte_api/__version__.py]

scrapy_zyte_api/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.12.0"

scrapy_zyte_api/handler.py

@@ -20,6 +20,7 @@
 
 from ._params import _ParamParser
 from .responses import ZyteAPIResponse, ZyteAPITextResponse, _process_response
+from .utils import USER_AGENT
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +130,7 @@ def _build_client(settings):
                 api_key=settings.get("ZYTE_API_KEY") or None,
                 api_url=settings.get("ZYTE_API_URL") or API_URL,
                 n_conn=settings.getint("CONCURRENT_REQUESTS"),
+                user_agent=settings.get("_ZYTE_API_USER_AGENT", default=USER_AGENT),
             )
         except NoApiKey:
             logger.warning(
@@ -208,6 +210,7 @@ async def _download_request(
         else:
             retrying = self._retry_policy
         self._log_request(api_params)
+
         try:
             api_response = await self._client.request_raw(
                 api_params,

scrapy_zyte_api/utils.py

@@ -1,5 +1,10 @@
 import scrapy
 from packaging.version import Version
+from zyte_api.utils import USER_AGENT as PYTHON_ZYTE_API_USER_AGENT
+
+from .__version__ import __version__
+
+USER_AGENT = f"scrapy-zyte-api/{__version__} {PYTHON_ZYTE_API_USER_AGENT}"
 
 _SCRAPY_VERSION = Version(scrapy.__version__)
 _SCRAPY_2_1_0 = Version("2.1.0")

setup.py

@@ -1,8 +1,19 @@
+import os
+
 import setuptools
 
+
+def get_version():
+    about = {}
+    here = os.path.abspath(os.path.dirname(__file__))
+    with open(os.path.join(here, "scrapy_zyte_api/__version__.py")) as f:
+        exec(f.read(), about)
+    return about["__version__"]
+
+
 setuptools.setup(
     name="scrapy-zyte-api",
-    version="0.12.0",
+    version=get_version(),
     description="Client library to process URLs through Zyte API",
     long_description=open("README.rst").read(),
     long_description_content_type="text/x-rst",
@@ -14,7 +25,7 @@
     install_requires=[
         "packaging>=20.0",
         "scrapy>=2.0.1",
-        "zyte-api>=0.4.0",
+        "zyte-api>=0.4.7",
     ],
     extras_require={
         # Sync with [testenv:provider-pinned] @ tox.ini

tests/test_handler.py

@@ -10,13 +10,15 @@
 from pytest_twisted import ensureDeferred
 from scrapy import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.settings import Settings
 from scrapy.utils.misc import create_instance
 from scrapy.utils.test import get_crawler
 from zyte_api.aio.client import AsyncClient
 from zyte_api.aio.retry import RetryFactory
 from zyte_api.constants import API_URL
 
 from scrapy_zyte_api.handler import ScrapyZyteAPIDownloadHandler
+from scrapy_zyte_api.utils import USER_AGENT
 
 from . import DEFAULT_CLIENT_CONCURRENCY, SETTINGS, UNSET, make_handler, set_env
 
@@ -456,3 +458,27 @@ def test_trust_env(enabled):
         crawler=crawler,
     )
     assert handler._session._trust_env == enabled
+
+
+@pytest.mark.parametrize(
+    "user_agent,expected",
+    (
+        (
+            None,
+            USER_AGENT,
+        ),
+        (
+            "zyte-crawlers/0.0.1",
+            "zyte-crawlers/0.0.1",
+        ),
+    ),
+)
+def test_user_agent_for_build_client(user_agent, expected):
+    settings = Settings(
+        {
+            **SETTINGS,
+            "_ZYTE_API_USER_AGENT": user_agent,
+        }
+    )
+    client = ScrapyZyteAPIDownloadHandler._build_client(settings)
+    assert client.user_agent == expected