Skip to content

Commit

Permalink
Extend the public API of session management (#211)
Browse files Browse the repository at this point in the history
  • Loading branch information
Gallaecio authored Jul 18, 2024
1 parent cd0aead commit 42e81f6
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 0 deletions.
12 changes: 12 additions & 0 deletions docs/usage/session.rst
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,18 @@ To define a different session config for a given URL pattern, install

.. autofunction:: scrapy_zyte_api.session_config

If in a session config implementation or in any other Scrapy component you need
to tell whether a request is a :ref:`session initialization request
<session-init>` or not, use :func:`~scrapy_zyte_api.is_session_init_request`:

.. autofunction:: scrapy_zyte_api.is_session_init_request

Classes decorated with :func:`~scrapy_zyte_api.session_config` are registered
into :data:`~scrapy_zyte_api.session_config_registry`:

.. autodata:: scrapy_zyte_api.session_config_registry
:annotation:

.. _session-cookies:

Cookie handling
Expand Down
6 changes: 6 additions & 0 deletions scrapy_zyte_api/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
from ._session import (
ScrapyZyteAPISessionDownloaderMiddleware,
SessionConfig,
is_session_init_request,
session_config,
)
from ._session import session_config_registry as _session_config_registry
from .addon import Addon
from .handler import ScrapyZyteAPIDownloadHandler

Expand All @@ -37,3 +39,7 @@
#: .. note:: When using python-zyte-api 0.5.2 or lower, this is the same as
#: :data:`~scrapy_zyte_api.SESSION_DEFAULT_RETRY_POLICY`.
SESSION_AGGRESSIVE_RETRY_POLICY = _SESSION_AGGRESSIVE_RETRY_POLICY

#: Instance of :class:`web_poet.rules.RulesRegistry` that holds :ref:`session
#: configs <session-configs>`.
session_config_registry = _session_config_registry
14 changes: 14 additions & 0 deletions scrapy_zyte_api/_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@
ZYTE_API_META_KEYS = ("zyte_api", "zyte_api_automap", "zyte_api_provider")


def is_session_init_request(request):
"""Return ``True`` if the request is a :ref:`session initialization request
<session-init>` or ``False`` otherwise."""
return request.meta.get(SESSION_INIT_META_KEY, False) is True


class SessionRetryFactory(RetryFactory):
temporary_download_error_stop = stop_after_attempt(1)

Expand Down Expand Up @@ -338,6 +344,10 @@ def params(self, request: Request) -> Dict[str, Any]:
"url": "https://example.com/new-session",
"httpResponseBody": True,
}
The returned parameters do not need to include :http:`request:url`. If
missing, it is picked from the request :ref:`triggering a session
initialization request <pool-size>`.
"""
if location := self.location(request):
return {
Expand All @@ -358,6 +368,10 @@ def check(self, response: Response, request: Request) -> bool:
The default implementation checks the outcome of the ``setLocation``
action if a location was defined, as described in :ref:`session-check`.
If you need to tell whether *request* is a :ref:`session initialization
request <session-init>` or not, use
:func:`~scrapy_zyte_api.is_session_init_request`.
"""
if self._checker:
return self._checker.check(response, request)
Expand Down
14 changes: 14 additions & 0 deletions tests/test_sessions.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
SESSION_AGGRESSIVE_RETRY_POLICY,
SESSION_DEFAULT_RETRY_POLICY,
SessionConfig,
is_session_init_request,
session_config,
)
from scrapy_zyte_api._session import SESSION_INIT_META_KEY, session_config_registry
Expand Down Expand Up @@ -2649,3 +2650,16 @@ def parse(self, response):
}
if reason is not None:
assert reason in caplog.text


@pytest.mark.parametrize(
("meta", "expected"),
(
({}, False),
({SESSION_INIT_META_KEY: False}, False),
({SESSION_INIT_META_KEY: True}, True),
),
)
def test_is_session_init_request(meta, expected):
actual = is_session_init_request(Request("https://example.com", meta=meta))
assert expected == actual

0 comments on commit 42e81f6

Please sign in to comment.