From 40f339ec832e64c3b59ac6375f29fad2575fa18e Mon Sep 17 00:00:00 2001 From: Robb Shecter Date: Mon, 18 Dec 2023 17:46:17 -0700 Subject: [PATCH] crs 2023 december (#201) * cleanup * checkpoint * checkpoint * checkpoint * type tweaks * fix: date parsing * scrapy typing * remove toolz typings * update vcr typings * checkpoint * checkpoint * checkpoint * checkpoint * checkpoint * refactor --- public_law/dates.py | 15 +- public_law/parsers/aus/ip_glossary.py | 2 +- public_law/parsers/can/doj_glossaries.py | 6 +- public_law/parsers/usa/colorado/crs.py | 45 +- .../parsers/usa/colorado/crs_articles.py | 4 - .../parsers/usa/colorado/crs_divisions.py | 6 +- .../parsers/usa/colorado/crs_sections.py | 10 +- public_law/parsers/usa/georgia_ag_opinions.py | 4 +- public_law/selector_util.py | 3 +- public_law/spiders/usa/colorado_crs.py | 22 +- public_law/spiders/usa/georgia_ag_opinions.py | 30 +- public_law/spiders/usa/oregon_regs.py | 2 +- pyproject.toml | 1 + script/create-crs-json | 3 + script/create-crs-xml-files-notes | 3 - typings/scrapy/__init__.pyi | 22 - typings/scrapy/__main__.pyi | 6 - typings/scrapy/addons.pyi | 30 - typings/scrapy/cmdline.pyi | 15 - typings/scrapy/commands/__init__.pyi | 97 --- typings/scrapy/contracts/__init__.pyi | 51 -- typings/scrapy/core/__init__.pyi | 7 - typings/scrapy/core/downloader/__init__.pyi | 62 -- .../core/downloader/handlers/__init__.pyi | 28 - typings/scrapy/core/downloader/middleware.pyi | 22 - typings/scrapy/core/engine.pyi | 82 --- typings/scrapy/core/scheduler.pyi | 217 ------ typings/scrapy/core/scraper.pyi | 71 -- typings/scrapy/core/spidermw.pyi | 31 - typings/scrapy/crawler.pyi | 146 ---- .../scrapy/downloadermiddlewares/__init__.pyi | 4 - typings/scrapy/dupefilters.pyi | 62 -- typings/scrapy/exceptions.pyi | 82 --- typings/scrapy/exporters.pyi | 131 ---- typings/scrapy/extension.pyi | 15 - typings/scrapy/extensions/__init__.pyi | 4 - typings/scrapy/http/__init__.pyi | 20 - typings/scrapy/http/common.pyi | 7 - typings/scrapy/http/headers.pyi | 61 -- typings/scrapy/http/request/__init__.pyi | 116 --- typings/scrapy/http/request/form.pyi | 28 - typings/scrapy/http/request/json_request.pyi | 28 - typings/scrapy/http/request/rpc.pyi | 20 - typings/scrapy/http/response/__init__.pyi | 111 --- typings/scrapy/http/response/html.pyi | 16 - typings/scrapy/http/response/text.pyi | 113 --- typings/scrapy/http/response/xml.pyi | 16 - typings/scrapy/interfaces.pyi | 27 - typings/scrapy/item.pyi | 92 --- typings/scrapy/link.pyi | 44 -- typings/scrapy/linkextractors/__init__.pyi | 16 - typings/scrapy/linkextractors/lxmlhtml.pyi | 41 -- typings/scrapy/loader/__init__.pyi | 88 --- typings/scrapy/logformatter.pyi | 98 --- typings/scrapy/mail.pyi | 24 - typings/scrapy/middleware.pyi | 36 - typings/scrapy/pipelines/__init__.pyi | 23 - typings/scrapy/pqueues.pyi | 122 ---- typings/scrapy/resolver.pyi | 78 -- typings/scrapy/responsetypes.pyi | 52 -- typings/scrapy/robotstxt.pyi | 91 --- typings/scrapy/selector/__init__.pyi | 9 - typings/scrapy/selector/unified.pyi | 59 -- typings/scrapy/settings/__init__.pyi | 371 ---------- typings/scrapy/settings/default_settings.pyi | 184 ----- typings/scrapy/shell.pyi | 35 - typings/scrapy/signalmanager.pyi | 68 -- typings/scrapy/signals.pyi | 34 - typings/scrapy/spiderloader.pyi | 47 -- typings/scrapy/spidermiddlewares/__init__.pyi | 4 - typings/scrapy/spiders/__init__.pyi | 75 -- typings/scrapy/spiders/crawl.pyi | 38 - typings/scrapy/spiders/feed.pyi | 89 --- typings/scrapy/spiders/sitemap.pyi | 33 - typings/scrapy/squeues.pyi | 17 - typings/scrapy/statscollectors.pyi | 78 -- typings/scrapy/utils/__init__.pyi | 4 - typings/scrapy/utils/asyncgen.pyi | 13 - typings/scrapy/utils/conf.pyi | 50 -- typings/scrapy/utils/console.pyi | 17 - typings/scrapy/utils/curl.pyi | 31 - typings/scrapy/utils/datatypes.pyi | 136 ---- typings/scrapy/utils/decorators.pyi | 23 - typings/scrapy/utils/defer.pyi | 206 ------ typings/scrapy/utils/deprecate.pyi | 82 --- typings/scrapy/utils/gz.pyi | 16 - typings/scrapy/utils/httpobj.pyi | 17 - typings/scrapy/utils/iterators.pyi | 51 -- typings/scrapy/utils/job.pyi | 10 - typings/scrapy/utils/log.pyi | 112 --- typings/scrapy/utils/misc.pyi | 112 --- typings/scrapy/utils/ossignal.pyi | 18 - typings/scrapy/utils/project.pyi | 25 - typings/scrapy/utils/python.pyi | 201 ------ typings/scrapy/utils/reactor.pyi | 61 -- typings/scrapy/utils/request.pyi | 150 ---- typings/scrapy/utils/response.pyi | 44 -- typings/scrapy/utils/serialize.pyi | 19 - typings/scrapy/utils/signal.pyi | 28 - typings/scrapy/utils/sitemap.pyi | 29 - typings/scrapy/utils/spider.pyi | 65 -- typings/scrapy/utils/ssl.pyi | 19 - typings/scrapy/utils/trackref.pyi | 46 -- typings/scrapy/utils/url.pyi | 83 --- typings/scrapy/utils/versions.pyi | 9 - typings/toolz/__init__.pyi | 17 - typings/toolz/_signatures.pyi | 73 -- typings/toolz/_version.pyi | 8 - typings/toolz/curried/__init__.pyi | 77 -- typings/toolz/curried/exceptions.pyi | 15 - typings/toolz/curried/operator.pyi | 5 - typings/toolz/dicttoolz.pyi | 235 ------ typings/toolz/functoolz.pyi | 540 -------------- typings/toolz/itertoolz.pyi | 666 ------------------ typings/toolz/recipes.pyi | 42 -- typings/toolz/sandbox/__init__.pyi | 7 - typings/toolz/sandbox/core.pyi | 104 --- typings/toolz/sandbox/parallel.pyi | 46 -- typings/toolz/utils.pyi | 8 - typings/vcr/__init__.pyi | 3 + typings/vcr/_handle_coroutine.pyi | 3 + typings/vcr/cassette.pyi | 39 +- typings/vcr/config.pyi | 19 +- typings/vcr/errors.pyi | 3 + typings/vcr/filters.pyi | 3 + typings/vcr/matchers.pyi | 27 +- typings/vcr/migration.pyi | 24 +- typings/vcr/patch.pyi | 18 +- typings/vcr/persisters/__init__.pyi | 3 + typings/vcr/record_mode.pyi | 3 + typings/vcr/request.pyi | 31 +- typings/vcr/serialize.pyi | 5 +- typings/vcr/serializers/__init__.pyi | 3 + typings/vcr/stubs/__init__.pyi | 72 +- typings/vcr/stubs/compat.pyi | 9 +- typings/vcr/util.pyi | 23 +- 136 files changed, 237 insertions(+), 7421 deletions(-) create mode 100755 script/create-crs-json delete mode 100644 typings/scrapy/__init__.pyi delete mode 100644 typings/scrapy/__main__.pyi delete mode 100644 typings/scrapy/addons.pyi delete mode 100644 typings/scrapy/cmdline.pyi delete mode 100644 typings/scrapy/commands/__init__.pyi delete mode 100644 typings/scrapy/contracts/__init__.pyi delete mode 100644 typings/scrapy/core/__init__.pyi delete mode 100644 typings/scrapy/core/downloader/__init__.pyi delete mode 100644 typings/scrapy/core/downloader/handlers/__init__.pyi delete mode 100644 typings/scrapy/core/downloader/middleware.pyi delete mode 100644 typings/scrapy/core/engine.pyi delete mode 100644 typings/scrapy/core/scheduler.pyi delete mode 100644 typings/scrapy/core/scraper.pyi delete mode 100644 typings/scrapy/core/spidermw.pyi delete mode 100644 typings/scrapy/crawler.pyi delete mode 100644 typings/scrapy/downloadermiddlewares/__init__.pyi delete mode 100644 typings/scrapy/dupefilters.pyi delete mode 100644 typings/scrapy/exceptions.pyi delete mode 100644 typings/scrapy/exporters.pyi delete mode 100644 typings/scrapy/extension.pyi delete mode 100644 typings/scrapy/extensions/__init__.pyi delete mode 100644 typings/scrapy/http/__init__.pyi delete mode 100644 typings/scrapy/http/common.pyi delete mode 100644 typings/scrapy/http/headers.pyi delete mode 100644 typings/scrapy/http/request/__init__.pyi delete mode 100644 typings/scrapy/http/request/form.pyi delete mode 100644 typings/scrapy/http/request/json_request.pyi delete mode 100644 typings/scrapy/http/request/rpc.pyi delete mode 100644 typings/scrapy/http/response/__init__.pyi delete mode 100644 typings/scrapy/http/response/html.pyi delete mode 100644 typings/scrapy/http/response/text.pyi delete mode 100644 typings/scrapy/http/response/xml.pyi delete mode 100644 typings/scrapy/interfaces.pyi delete mode 100644 typings/scrapy/item.pyi delete mode 100644 typings/scrapy/link.pyi delete mode 100644 typings/scrapy/linkextractors/__init__.pyi delete mode 100644 typings/scrapy/linkextractors/lxmlhtml.pyi delete mode 100644 typings/scrapy/loader/__init__.pyi delete mode 100644 typings/scrapy/logformatter.pyi delete mode 100644 typings/scrapy/mail.pyi delete mode 100644 typings/scrapy/middleware.pyi delete mode 100644 typings/scrapy/pipelines/__init__.pyi delete mode 100644 typings/scrapy/pqueues.pyi delete mode 100644 typings/scrapy/resolver.pyi delete mode 100644 typings/scrapy/responsetypes.pyi delete mode 100644 typings/scrapy/robotstxt.pyi delete mode 100644 typings/scrapy/selector/__init__.pyi delete mode 100644 typings/scrapy/selector/unified.pyi delete mode 100644 typings/scrapy/settings/__init__.pyi delete mode 100644 typings/scrapy/settings/default_settings.pyi delete mode 100644 typings/scrapy/shell.pyi delete mode 100644 typings/scrapy/signalmanager.pyi delete mode 100644 typings/scrapy/signals.pyi delete mode 100644 typings/scrapy/spiderloader.pyi delete mode 100644 typings/scrapy/spidermiddlewares/__init__.pyi delete mode 100644 typings/scrapy/spiders/__init__.pyi delete mode 100644 typings/scrapy/spiders/crawl.pyi delete mode 100644 typings/scrapy/spiders/feed.pyi delete mode 100644 typings/scrapy/spiders/sitemap.pyi delete mode 100644 typings/scrapy/squeues.pyi delete mode 100644 typings/scrapy/statscollectors.pyi delete mode 100644 typings/scrapy/utils/__init__.pyi delete mode 100644 typings/scrapy/utils/asyncgen.pyi delete mode 100644 typings/scrapy/utils/conf.pyi delete mode 100644 typings/scrapy/utils/console.pyi delete mode 100644 typings/scrapy/utils/curl.pyi delete mode 100644 typings/scrapy/utils/datatypes.pyi delete mode 100644 typings/scrapy/utils/decorators.pyi delete mode 100644 typings/scrapy/utils/defer.pyi delete mode 100644 typings/scrapy/utils/deprecate.pyi delete mode 100644 typings/scrapy/utils/gz.pyi delete mode 100644 typings/scrapy/utils/httpobj.pyi delete mode 100644 typings/scrapy/utils/iterators.pyi delete mode 100644 typings/scrapy/utils/job.pyi delete mode 100644 typings/scrapy/utils/log.pyi delete mode 100644 typings/scrapy/utils/misc.pyi delete mode 100644 typings/scrapy/utils/ossignal.pyi delete mode 100644 typings/scrapy/utils/project.pyi delete mode 100644 typings/scrapy/utils/python.pyi delete mode 100644 typings/scrapy/utils/reactor.pyi delete mode 100644 typings/scrapy/utils/request.pyi delete mode 100644 typings/scrapy/utils/response.pyi delete mode 100644 typings/scrapy/utils/serialize.pyi delete mode 100644 typings/scrapy/utils/signal.pyi delete mode 100644 typings/scrapy/utils/sitemap.pyi delete mode 100644 typings/scrapy/utils/spider.pyi delete mode 100644 typings/scrapy/utils/ssl.pyi delete mode 100644 typings/scrapy/utils/trackref.pyi delete mode 100644 typings/scrapy/utils/url.pyi delete mode 100644 typings/scrapy/utils/versions.pyi delete mode 100644 typings/toolz/__init__.pyi delete mode 100644 typings/toolz/_signatures.pyi delete mode 100644 typings/toolz/_version.pyi delete mode 100644 typings/toolz/curried/__init__.pyi delete mode 100644 typings/toolz/curried/exceptions.pyi delete mode 100644 typings/toolz/curried/operator.pyi delete mode 100644 typings/toolz/dicttoolz.pyi delete mode 100644 typings/toolz/functoolz.pyi delete mode 100644 typings/toolz/itertoolz.pyi delete mode 100644 typings/toolz/recipes.pyi delete mode 100644 typings/toolz/sandbox/__init__.pyi delete mode 100644 typings/toolz/sandbox/core.pyi delete mode 100644 typings/toolz/sandbox/parallel.pyi delete mode 100644 typings/toolz/utils.pyi diff --git a/public_law/dates.py b/public_law/dates.py index 00a6e1f8..56278770 100644 --- a/public_law/dates.py +++ b/public_law/dates.py @@ -1,3 +1,5 @@ +"""Provide date-related functions.""" + from typing import Protocol from datetime import datetime, date @@ -17,6 +19,12 @@ def todays_date() -> str: return iso_8601(today()) +def current_year() -> int: + """Provide the current year.""" + + return today().year + + def today() -> date: """Provide today's date in the given timezone.""" @@ -24,10 +32,9 @@ def today() -> date: # But the Scrapy settings don't seem to be # available in this context. # See https://doc.scrapy.org/en/latest/topics/settings.html. - mountain = pytz.timezone( - "US/Mountain" - ) - return mountain.localize(datetime.now()).date() + tz = pytz.timezone("US/Mountain") + + return tz.localize(datetime.now()).date() def iso_8601(a_date: date) -> str: diff --git a/public_law/parsers/aus/ip_glossary.py b/public_law/parsers/aus/ip_glossary.py index af170ba0..63c7148f 100644 --- a/public_law/parsers/aus/ip_glossary.py +++ b/public_law/parsers/aus/ip_glossary.py @@ -60,6 +60,6 @@ def _parse_mod_date(html: HtmlResponse) -> date: 26 March 2021 """ mod_date_str: str = cast(str, ( - html.selector.css("span.date-display-single").xpath("@content").get() # type: ignore + html.selector.css("span.date-display-single").xpath("@content").get() )) return datetime.fromisoformat(mod_date_str).date() diff --git a/public_law/parsers/can/doj_glossaries.py b/public_law/parsers/can/doj_glossaries.py index 3a116e32..5b4e25fd 100644 --- a/public_law/parsers/can/doj_glossaries.py +++ b/public_law/parsers/can/doj_glossaries.py @@ -102,9 +102,9 @@ } -def configured_urls() -> tuple[str, ...]: +def configured_urls() -> list[str]: """All the URLs that have been properly set up with subjects.""" - return tuple(SUBJECTS.keys()) + return list(SUBJECTS.keys()) def parse_glossary(html: HtmlResponse) -> GlossaryParseResult: @@ -114,7 +114,7 @@ def parse_glossary(html: HtmlResponse) -> GlossaryParseResult: entries: list[GlossaryEntry] = [] match html.css("main dl"): - case [first, *_] if isinstance(first, Selector): + case [first, *_]: first_dl_list = first case _: raise ParseException("Expected a
") diff --git a/public_law/parsers/usa/colorado/crs.py b/public_law/parsers/usa/colorado/crs.py index 5255a3a1..c30c03b6 100644 --- a/public_law/parsers/usa/colorado/crs.py +++ b/public_law/parsers/usa/colorado/crs.py @@ -1,12 +1,7 @@ # pyright: reportUnknownMemberType=false -# pyright: reportOptionalMemberAccess=false -# pyright: reportUnknownVariableType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownLambdaType=false - from scrapy.selector.unified import Selector -from scrapy.http.response import Response +from scrapy.http.response.xml import XmlResponse from typing import Any @@ -16,23 +11,29 @@ from public_law.parsers.usa.colorado.crs_divisions import parse_divisions -def parse_title_bang(dom: Response, logger: Any) -> Title: +def parse_title_bang(dom: XmlResponse, logger: Any) -> Title: match parse_title(dom, logger): case None: raise Exception("Could not parse title") - case result: - return result + case title: + return title -def parse_title(dom: Response, logger: Any) -> Title | None: - raw_name = dom.xpath("//TITLE-TEXT/text()").get() - - if raw_name is None: - logger.warn(f"Could not parse title name in {dom.url}") - return None +def parse_title(dom: XmlResponse, logger: Any) -> Title | None: + match(dom.xpath("//TITLE-TEXT/text()").get()): + case str(raw_name): + name = NonemptyString(titleize(raw_name)) + case None: + logger.warn(f"Could not the parse title name in {dom.url}") + return None + + match(dom.xpath("//TITLE-NUM/text()").get()): + case str(raw_number): + number = NonemptyString(raw_number.split(" ")[1]) + case None: + logger.warn(f"Could not the parse title number in {dom.url}") + return None - name = NonemptyString(titleize(raw_name)) - number = NonemptyString(dom.xpath("//TITLE-NUM/text()").get().split(" ")[1]) url_number = number.rjust(2, "0") source_url = URL(f"https://leg.colorado.gov/sites/default/files/images/olls/crs2022-title-{url_number}.pdf") @@ -44,13 +45,15 @@ def parse_title(dom: Response, logger: Any) -> Title | None: ) -def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Division] | list[Article]: +def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | XmlResponse, logger: Any) -> list[Division] | list[Article]: division_nodes = dom.xpath("//T-DIV") article_nodes = dom.xpath("//TA-LIST") if len(division_nodes) > 0: - return parse_divisions(title_number, dom, logger) + func = parse_divisions elif len(article_nodes) > 0: - return parse_articles(title_number, dom, logger) + func = parse_articles else: - raise Exception(f"Could not parse divisions or articles in Title {title_number}") + raise Exception(f"Could not parse divisions or articles in Title {title_number}. Neither T-DIV nor TA-LIST nodes were found.") + + return func(title_number, dom, logger) diff --git a/public_law/parsers/usa/colorado/crs_articles.py b/public_law/parsers/usa/colorado/crs_articles.py index fab29429..ac7d18ae 100644 --- a/public_law/parsers/usa/colorado/crs_articles.py +++ b/public_law/parsers/usa/colorado/crs_articles.py @@ -1,8 +1,4 @@ # pyright: reportUnknownMemberType=false -# pyright: reportOptionalMemberAccess=false -# pyright: reportUnknownVariableType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownLambdaType=false from itertools import takewhile, dropwhile from typing import Any diff --git a/public_law/parsers/usa/colorado/crs_divisions.py b/public_law/parsers/usa/colorado/crs_divisions.py index 1f4239e8..ec8ba318 100644 --- a/public_law/parsers/usa/colorado/crs_divisions.py +++ b/public_law/parsers/usa/colorado/crs_divisions.py @@ -1,8 +1,4 @@ # pyright: reportUnknownMemberType=false -# pyright: reportOptionalMemberAccess=false -# pyright: reportUnknownVariableType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownLambdaType=false from scrapy.selector.unified import Selector @@ -22,7 +18,7 @@ def parse_divisions(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Division]: division_nodes = dom.xpath("//T-DIV") - divs = [] + divs: list[Division] = [] for div_node in division_nodes: raw_div_name = div_name_text(div_node) diff --git a/public_law/parsers/usa/colorado/crs_sections.py b/public_law/parsers/usa/colorado/crs_sections.py index 9b079f76..723c6c0c 100644 --- a/public_law/parsers/usa/colorado/crs_sections.py +++ b/public_law/parsers/usa/colorado/crs_sections.py @@ -1,13 +1,9 @@ # pyright: reportUnknownMemberType=false -# pyright: reportOptionalMemberAccess=false -# pyright: reportUnknownVariableType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownLambdaType=false from typing import Any from bs4 import BeautifulSoup -from scrapy.http.response import Response +from scrapy.http.response.xml import XmlResponse from scrapy.selector.unified import Selector from public_law.selector_util import just_text @@ -16,10 +12,10 @@ -def parse_sections(dom: Response, logger: Any) -> list[Section]: +def parse_sections(dom: XmlResponse, logger: Any) -> list[Section]: section_nodes = dom.xpath("//SECTION-TEXT") - sections = [] + sections: list[Section] = [] for node in section_nodes: if _is_repealed(node): continue diff --git a/public_law/parsers/usa/georgia_ag_opinions.py b/public_law/parsers/usa/georgia_ag_opinions.py index 8e0ee117..d85eec67 100644 --- a/public_law/parsers/usa/georgia_ag_opinions.py +++ b/public_law/parsers/usa/georgia_ag_opinions.py @@ -48,12 +48,12 @@ def parse_ag_opinion(html: Response) -> OpinionParseResult: join("\n"), ), ) - citation_set = pipe( + citation_set = cast(CitationSet, pipe( re.findall(r"\d+-\d+-\d+(?:\([-().A-Za-z0-9]*[-A-Za-z0-9]\))?", full_text), set, sorted, CitationSet, - ) + )) return OpinionParseResult( summary=summary, diff --git a/public_law/selector_util.py b/public_law/selector_util.py index d85bdc62..65783a9a 100644 --- a/public_law/selector_util.py +++ b/public_law/selector_util.py @@ -3,8 +3,9 @@ from scrapy.selector.unified import Selector, SelectorList -def node_name(node: Selector): +def node_name(node: Selector) -> str | None: return node.xpath("name()").get() + def just_text(node: Selector | SelectorList | Any) -> str | None: return node.xpath("text()").get() diff --git a/public_law/spiders/usa/colorado_crs.py b/public_law/spiders/usa/colorado_crs.py index d3d66044..c960ae82 100644 --- a/public_law/spiders/usa/colorado_crs.py +++ b/public_law/spiders/usa/colorado_crs.py @@ -1,21 +1,17 @@ # pyright: reportUnknownMemberType=false -# pyright: reportUnknownArgumentType=false # pyright: reportUnknownVariableType=false -# pyright: reportUnknownParameterType=false # pyright: reportGeneralTypeIssues=false -# pyright: reportUnusedCallResult=false import os -import re - from pathlib import Path +from typing import Any from progressbar import ProgressBar from scrapy import Spider from scrapy.http.request import Request from scrapy.http.response.html import HtmlResponse -from typing import Any +from public_law import dates from public_law.parsers.usa.colorado.crs import parse_title from public_law.parsers.usa.colorado.crs_sections import parse_sections @@ -25,7 +21,7 @@ class ColoradoCRS(Spider): Reads the sources from a local directory instead of the web. """ - name = "usa_colorado_crs" + name = "usa_colorado_crs" def start_requests(self): @@ -53,21 +49,11 @@ def start_requests(self): def parse(self, response: HtmlResponse, **_: dict[str, Any]): # type: ignore[override] if "README.txt" in response.url: - yield from self.parse_readme(response) + yield { "kind": "CRS", "edition": dates.current_year() } else: yield from self.parse_title_xml(response) - def parse_readme(self, response: HtmlResponse, **_: dict[str, Any]): - result = re.findall(r'COLORADO REVISED STATUTES (\d\d\d\d) DATASET', str(response.body)) - if len(result) != 1: - raise Exception(f"Could not parse year from README: {response.body}") - - year: str = result[0] - - yield { "kind": "CRS", "edition": int(year) } - - def parse_title_xml(self, response: HtmlResponse, **_: dict[str, Any]): """Framework callback which parses one XML file.""" self.logger.debug(f"Parsing {response.url}...") diff --git a/public_law/spiders/usa/georgia_ag_opinions.py b/public_law/spiders/usa/georgia_ag_opinions.py index 2b4fde10..2f63102e 100644 --- a/public_law/spiders/usa/georgia_ag_opinions.py +++ b/public_law/spiders/usa/georgia_ag_opinions.py @@ -6,11 +6,13 @@ # pyright: reportGeneralTypeIssues=false -from ...parsers.usa.georgia_ag_opinions import parse_ag_opinion from scrapy import Spider from scrapy.http.request import Request from scrapy.http.response import Response -from typing import Any, Dict, cast +from scrapy.http.response.html import HtmlResponse +from typing import Any, Dict + +from ...parsers.usa.georgia_ag_opinions import parse_ag_opinion JD_VERBOSE_NAME = "USA / Georgia" PUBLICATION_NAME = "Attorney General Opinions" @@ -29,23 +31,28 @@ class GeorgiaAgOpinions(Spider): "https://law.georgia.gov/opinions/unofficial", ] + def parse(self, response: Response, **kwargs: Dict[str, Any]): """Framework callback which begins the parsing.""" - return self.parse_index_page(response) - def parse_index_page(self, response: Response): + match(response): + case HtmlResponse(): + return self.parse_index_page(response) + + case _: + raise Exception(f"Unexpected response type: {type(response)}") + + + def parse_index_page(self, response: HtmlResponse): # # 1. Find all the individual opinions on this index page # and request a parse for each. # - opinion_paths = cast( - list[str], - response.xpath( + opinion_paths = response.xpath( "//td[contains(@class, 'views-field-title')]/a/@href" - ).getall(), - ) + ).getall() - for url in [cast(str, response.urljoin(p)) for p in opinion_paths]: + for url in [response.urljoin(p) for p in opinion_paths]: yield Request(url, callback=self.parse_opinion_page) # @@ -60,5 +67,6 @@ def parse_index_page(self, response: Response): response.urljoin(next_page_path), callback=self.parse_index_page ) - def parse_opinion_page(self, response: Response): + + def parse_opinion_page(self, response: HtmlResponse): yield parse_ag_opinion(response)._asdict() diff --git a/public_law/spiders/usa/oregon_regs.py b/public_law/spiders/usa/oregon_regs.py index a3149b7e..8c22b9fa 100644 --- a/public_law/spiders/usa/oregon_regs.py +++ b/public_law/spiders/usa/oregon_regs.py @@ -106,7 +106,7 @@ def from_crawler(cls, crawler: Crawler, *args: List[str], **kwargs: Dict[str, An """Override to register to receive the idle event""" spider = cast(OregonRegs, super(OregonRegs, cls).from_crawler(crawler, *args, **kwargs)) - crawler.signals.connect(spider.spider_idle, signal=scrapy.signals.spider_idle) # type: ignore + crawler.signals.connect(spider.spider_idle, signal=scrapy.signals.spider_idle) return spider def spider_idle(self, spider: Spider): diff --git a/pyproject.toml b/pyproject.toml index af1a2a44..5028fd08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ typeCheckingMode = "strict" reportCallInDefaultInitializer = "error" reportImplicitStringConcatenation = "error" reportMissingSuperCall = "error" +reportMissingTypeStubs = false reportPropertyTypeMismatch = "error" reportUninitializedInstanceVariable = "error" reportUnnecessaryTypeIgnoreComment = "error" diff --git a/script/create-crs-json b/script/create-crs-json new file mode 100755 index 00000000..6352824c --- /dev/null +++ b/script/create-crs-json @@ -0,0 +1,3 @@ +#!/usr/bin/env fish + +scrapy crawl -a crsdata_dir=tmp/sources --overwrite-output tmp/crs.json:jsonl usa_colorado_crs diff --git a/script/create-crs-xml-files-notes b/script/create-crs-xml-files-notes index c0110b8e..1518575d 100755 --- a/script/create-crs-xml-files-notes +++ b/script/create-crs-xml-files-notes @@ -1,8 +1,5 @@ #!/usr/bin/env fish -# -# So far, just a history of commands I ran. -# cd tmp/sources/TITLES diff --git a/typings/scrapy/__init__.pyi b/typings/scrapy/__init__.pyi deleted file mode 100644 index f8bc8ba8..00000000 --- a/typings/scrapy/__init__.pyi +++ /dev/null @@ -1,22 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import pkgutil -import sys -import warnings -from twisted import version as _txv -from scrapy.http import FormRequest, Request -from scrapy.item import Field, Item -from scrapy.selector import Selector -from scrapy.spiders import Spider - -""" -Scrapy - a web crawling and web scraping framework written for Python -""" -__all__ = ["__version__", "version_info", "twisted_version", "Spider", "Request", "FormRequest", "Selector", "Item", "Field"] -__version__ = ... -version_info = ... -twisted_version = ... -if sys.version_info < (3, 8): - ... diff --git a/typings/scrapy/__main__.pyi b/typings/scrapy/__main__.pyi deleted file mode 100644 index cdba0a06..00000000 --- a/typings/scrapy/__main__.pyi +++ /dev/null @@ -1,6 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -if __name__ == "__main__": - ... diff --git a/typings/scrapy/addons.pyi b/typings/scrapy/addons.pyi deleted file mode 100644 index ce44131a..00000000 --- a/typings/scrapy/addons.pyi +++ /dev/null @@ -1,30 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import TYPE_CHECKING -from scrapy.settings import Settings -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -logger = ... -class AddonManager: - """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: Crawler) -> None: - ... - - def load_settings(self, settings: Settings) -> None: - """Load add-ons and configurations from a settings object and apply them. - - This will load the add-on for every add-on path in the - ``ADDONS`` setting and execute their ``update_settings`` methods. - - :param settings: The :class:`~scrapy.settings.Settings` object from \ - which to read the add-on configuration - :type settings: :class:`~scrapy.settings.Settings` - """ - ... - - - diff --git a/typings/scrapy/cmdline.pyi b/typings/scrapy/cmdline.pyi deleted file mode 100644 index 8cd98b9a..00000000 --- a/typings/scrapy/cmdline.pyi +++ /dev/null @@ -1,15 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import argparse - -class ScrapyArgumentParser(argparse.ArgumentParser): - ... - - -def execute(argv=..., settings=...): - ... - -if __name__ == "__main__": - ... diff --git a/typings/scrapy/commands/__init__.pyi b/typings/scrapy/commands/__init__.pyi deleted file mode 100644 index eccc4237..00000000 --- a/typings/scrapy/commands/__init__.pyi +++ /dev/null @@ -1,97 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import argparse -import os -from pathlib import Path -from typing import Any, Dict, List, Optional -from twisted.python import failure -from scrapy.crawler import CrawlerProcess -from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli - -""" -Base class for Scrapy commands -""" -class ScrapyCommand: - requires_project = ... - crawler_process: Optional[CrawlerProcess] = ... - default_settings: Dict[str, Any] = ... - exitcode = ... - def __init__(self) -> None: - ... - - def set_crawler(self, crawler): # -> None: - ... - - def syntax(self): # -> Literal['']: - """ - Command syntax (preferably one-line). Do not include command name. - """ - ... - - def short_desc(self): # -> Literal['']: - """ - A short description of the command - """ - ... - - def long_desc(self): # -> Literal['']: - """A long description of the command. Return short description when not - available. It cannot contain newlines since contents will be formatted - by optparser which removes newlines and wraps text. - """ - ... - - def help(self): # -> Literal['']: - """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines since no post-formatting will - be applied to its contents. - """ - ... - - def add_options(self, parser): # -> None: - """ - Populate option parse with options available for this command - """ - ... - - def process_options(self, args, opts): # -> None: - ... - - def run(self, args: List[str], opts: argparse.Namespace) -> None: - """ - Entry point for running commands - """ - ... - - - -class BaseRunSpiderCommand(ScrapyCommand): - """ - Common class used to share functionality between the crawl, parse and runspider commands - """ - def add_options(self, parser): # -> None: - ... - - def process_options(self, args, opts): # -> None: - ... - - - -class ScrapyHelpFormatter(argparse.HelpFormatter): - """ - Help Formatter for scrapy command line help messages. - """ - def __init__(self, prog, indent_increment=..., max_help_position=..., width=...) -> None: - ... - - def format_part_strings(self, part_strings): - """ - Underline and title case command line help message headers. - """ - ... - - - diff --git a/typings/scrapy/contracts/__init__.pyi b/typings/scrapy/contracts/__init__.pyi deleted file mode 100644 index eea3d100..00000000 --- a/typings/scrapy/contracts/__init__.pyi +++ /dev/null @@ -1,51 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import re -import sys -from functools import wraps -from inspect import getmembers -from types import CoroutineType -from typing import AsyncGenerator, Dict -from unittest import TestCase -from scrapy.http import Request -from scrapy.utils.python import get_spec -from scrapy.utils.spider import iterate_spider_output - -class Contract: - """Abstract class for contracts""" - request_cls = ... - def __init__(self, method, *args) -> None: - ... - - def add_pre_hook(self, request, results): - ... - - def add_post_hook(self, request, results): - ... - - def adjust_request_args(self, args): - ... - - - -class ContractsManager: - contracts: Dict[str, Contract] = ... - def __init__(self, contracts) -> None: - ... - - def tested_methods_from_spidercls(self, spidercls): # -> list[Any]: - ... - - def extract_contracts(self, method): # -> list[Any]: - ... - - def from_spider(self, spider, results): # -> list[Any]: - ... - - def from_method(self, method, results): # -> Request | None: - ... - - - diff --git a/typings/scrapy/core/__init__.pyi b/typings/scrapy/core/__init__.pyi deleted file mode 100644 index a0e30e24..00000000 --- a/typings/scrapy/core/__init__.pyi +++ /dev/null @@ -1,7 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Scrapy core library classes and functions. -""" diff --git a/typings/scrapy/core/downloader/__init__.pyi b/typings/scrapy/core/downloader/__init__.pyi deleted file mode 100644 index 1387ada3..00000000 --- a/typings/scrapy/core/downloader/__init__.pyi +++ /dev/null @@ -1,62 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import random -from collections import deque -from datetime import datetime -from time import time -from typing import Any, Deque, Dict, Set, TYPE_CHECKING, Tuple, cast -from twisted.internet import task -from twisted.internet.defer import Deferred -from scrapy import Request, Spider, signals -from scrapy.core.downloader.handlers import DownloadHandlers -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.http import Response -from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings -from scrapy.signalmanager import SignalManager -from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.httpobj import urlparse_cached -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -class Slot: - """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool) -> None: - ... - - def free_transfer_slots(self) -> int: - ... - - def download_delay(self) -> float: - ... - - def close(self) -> None: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... - - - -class Downloader: - DOWNLOAD_SLOT = ... - def __init__(self, crawler: Crawler) -> None: - ... - - def fetch(self, request: Request, spider: Spider) -> Deferred: - ... - - def needs_backout(self) -> bool: - ... - - def close(self) -> None: - ... - - - diff --git a/typings/scrapy/core/downloader/handlers/__init__.pyi b/typings/scrapy/core/downloader/handlers/__init__.pyi deleted file mode 100644 index 4d980648..00000000 --- a/typings/scrapy/core/downloader/handlers/__init__.pyi +++ /dev/null @@ -1,28 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import logging -from typing import Any, Callable, Dict, Generator, TYPE_CHECKING, Union, cast -from twisted.internet import defer -from twisted.internet.defer import Deferred -from scrapy import Request, Spider, signals -from scrapy.exceptions import NotConfigured, NotSupported -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import without_none_values -from scrapy.crawler import Crawler - -"""Download handlers for different schemes""" -if TYPE_CHECKING: - ... -logger = ... -class DownloadHandlers: - def __init__(self, crawler: Crawler) -> None: - ... - - def download_request(self, request: Request, spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/core/downloader/middleware.pyi b/typings/scrapy/core/downloader/middleware.pyi deleted file mode 100644 index 9bac3668..00000000 --- a/typings/scrapy/core/downloader/middleware.pyi +++ /dev/null @@ -1,22 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Callable -from twisted.internet.defer import Deferred -from scrapy import Spider -from scrapy.http import Request -from scrapy.middleware import MiddlewareManager - -""" -Downloader Middleware manager - -See documentation in docs/topics/downloader-middleware.rst -""" -class DownloaderMiddlewareManager(MiddlewareManager): - component_name = ... - def download(self, download_func: Callable, request: Request, spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/core/engine.pyi b/typings/scrapy/core/engine.pyi deleted file mode 100644 index 0d627102..00000000 --- a/typings/scrapy/core/engine.pyi +++ /dev/null @@ -1,82 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Callable, Generator, Iterable, TYPE_CHECKING -from twisted.internet.defer import Deferred, inlineCallbacks -from scrapy.http import Request -from scrapy.spiders import Spider -from scrapy.utils.reactor import CallLaterOnce -from scrapy.core.scheduler import BaseScheduler -from scrapy.crawler import Crawler - -""" -This is the Scrapy engine which controls the Scheduler, Downloader and Spider. - -For more information see docs/topics/architecture.rst - -""" -if TYPE_CHECKING: - ... -logger = ... -class Slot: - def __init__(self, start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce, scheduler: BaseScheduler) -> None: - ... - - def add_request(self, request: Request) -> None: - ... - - def remove_request(self, request: Request) -> None: - ... - - def close(self) -> Deferred: - ... - - - -class ExecutionEngine: - def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: - ... - - @inlineCallbacks - def start(self) -> Generator[Deferred, Any, None]: - ... - - def stop(self) -> Deferred: - """Gracefully stop the execution engine""" - ... - - def close(self) -> Deferred: - """ - Gracefully close the execution engine. - If it has already been started, stop it. In all cases, close the spider and the downloader. - """ - ... - - def pause(self) -> None: - ... - - def unpause(self) -> None: - ... - - def spider_is_idle(self) -> bool: - ... - - def crawl(self, request: Request) -> None: - """Inject the request into the spider <-> downloader pipeline""" - ... - - def download(self, request: Request) -> Deferred: - """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - ... - - @inlineCallbacks - def open_spider(self, spider: Spider, start_requests: Iterable = ..., close_if_idle: bool = ...) -> Generator[Deferred, Any, None]: - ... - - def close_spider(self, spider: Spider, reason: str = ...) -> Deferred: - """Close (cancel) spider and clear all its outstanding requests""" - ... - - - diff --git a/typings/scrapy/core/scheduler.pyi b/typings/scrapy/core/scheduler.pyi deleted file mode 100644 index c04f8c12..00000000 --- a/typings/scrapy/core/scheduler.pyi +++ /dev/null @@ -1,217 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from abc import abstractmethod -from typing import Any, Optional, TYPE_CHECKING, Type, TypeVar -from twisted.internet.defer import Deferred -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector -from typing_extensions import Self - -if TYPE_CHECKING: - ... -logger = ... -class BaseSchedulerMeta(type): - """ - Metaclass to check scheduler classes against the necessary interface - """ - def __instancecheck__(cls, instance: Any) -> bool: - ... - - def __subclasscheck__(cls, subclass: type) -> bool: - ... - - - -class BaseScheduler(metaclass=BaseSchedulerMeta): - """ - The scheduler component is responsible for storing requests received from - the engine, and feeding them back upon request (also to the engine). - - The original sources of said requests are: - - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks - * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods - * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods - - The order in which the scheduler returns its stored requests (via the ``next_request`` method) - plays a great part in determining the order in which those requests are downloaded. - - The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. - """ - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - """ - Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. - """ - ... - - def open(self, spider: Spider) -> Optional[Deferred]: - """ - Called when the spider is opened by the engine. It receives the spider - instance as argument and it's useful to execute initialization code. - - :param spider: the spider object for the current crawl - :type spider: :class:`~scrapy.spiders.Spider` - """ - ... - - def close(self, reason: str) -> Optional[Deferred]: - """ - Called when the spider is closed by the engine. It receives the reason why the crawl - finished as argument and it's useful to execute cleaning code. - - :param reason: a string which describes the reason why the spider was closed - :type reason: :class:`str` - """ - ... - - @abstractmethod - def has_pending_requests(self) -> bool: - """ - ``True`` if the scheduler has enqueued requests, ``False`` otherwise - """ - ... - - @abstractmethod - def enqueue_request(self, request: Request) -> bool: - """ - Process a request received by the engine. - - Return ``True`` if the request is stored correctly, ``False`` otherwise. - - If ``False``, the engine will fire a ``request_dropped`` signal, and - will not make further attempts to schedule the request at a later time. - For reference, the default Scrapy scheduler returns ``False`` when the - request is rejected by the dupefilter. - """ - ... - - @abstractmethod - def next_request(self) -> Optional[Request]: - """ - Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` - to indicate that there are no requests to be considered ready at the moment. - - Returning ``None`` implies that no request from the scheduler will be sent - to the downloader in the current reactor cycle. The engine will continue - calling ``next_request`` until ``has_pending_requests`` is ``False``. - """ - ... - - - -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") -class Scheduler(BaseScheduler): - """ - Default Scrapy scheduler. This implementation also handles duplication - filtering via the :setting:`dupefilter `. - - This scheduler stores requests into several priority queues (defined by the - :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues - are backed by either memory or disk based queues (respectively defined by the - :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). - - Request prioritization is almost entirely delegated to the priority queue. The only - prioritization performed by this scheduler is using the disk-based queue if present - (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based - queue if a serialization error occurs. If the disk queue is not present, the memory one - is used directly. - - :param dupefilter: An object responsible for checking and filtering duplicate requests. - The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. - :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: - any class that implements the `BaseDupeFilter` interface - - :param jobdir: The path of a directory to be used for persisting the crawl's state. - The value for the :setting:`JOBDIR` setting is used by default. - See :ref:`topics-jobs`. - :type jobdir: :class:`str` or ``None`` - - :param dqclass: A class to be used as persistent request queue. - The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. - :type dqclass: class - - :param mqclass: A class to be used as non-persistent request queue. - The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. - :type mqclass: class - - :param logunser: A boolean that indicates whether or not unserializable requests should be logged. - The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. - :type logunser: bool - - :param stats: A stats collector object to record stats about the request scheduling process. - The value for the :setting:`STATS_CLASS` setting is used by default. - :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: - any class that implements the `StatsCollector` interface - - :param pqclass: A class to be used as priority queue for requests. - The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. - :type pqclass: class - - :param crawler: The crawler object corresponding to the current crawl. - :type crawler: :class:`scrapy.crawler.Crawler` - """ - def __init__(self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = ..., dqclass=..., mqclass=..., logunser: bool = ..., stats: Optional[StatsCollector] = ..., pqclass=..., crawler: Optional[Crawler] = ...) -> None: - ... - - @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: - """ - Factory method, initializes the scheduler with arguments taken from the crawl settings - """ - ... - - def has_pending_requests(self) -> bool: - ... - - def open(self, spider: Spider) -> Optional[Deferred]: - """ - (1) initialize the memory queue - (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory - (3) return the result of the dupefilter's ``open`` method - """ - ... - - def close(self, reason: str) -> Optional[Deferred]: - """ - (1) dump pending requests to disk if there is a disk queue - (2) return the result of the dupefilter's ``close`` method - """ - ... - - def enqueue_request(self, request: Request) -> bool: - """ - Unless the received request is filtered out by the Dupefilter, attempt to push - it into the disk queue, falling back to pushing it into the memory queue. - - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. - - Return ``True`` if the request was stored successfully, ``False`` otherwise. - """ - ... - - def next_request(self) -> Optional[Request]: - """ - Return a :class:`~scrapy.http.Request` object from the memory queue, - falling back to the disk queue if the memory queue is empty. - Return ``None`` if there are no more enqueued requests. - - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. - """ - ... - - def __len__(self) -> int: - """ - Return the total amount of enqueued requests - """ - ... - - - diff --git a/typings/scrapy/core/scraper.pyi b/typings/scrapy/core/scraper.pyi deleted file mode 100644 index 46356f7c..00000000 --- a/typings/scrapy/core/scraper.pyi +++ /dev/null @@ -1,71 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, AsyncIterable, Generator, Iterable, TYPE_CHECKING, Tuple, Union -from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.http import Request, Response -from scrapy.crawler import Crawler - -"""This module implements the Scraper component which parses responses and -extracts information from them""" -if TYPE_CHECKING: - ... -QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] -logger = ... -class Slot: - """Scraper slot (one per running spider)""" - MIN_RESPONSE_SIZE = ... - def __init__(self, max_active_size: int = ...) -> None: - ... - - def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: - ... - - def next_response_request_deferred(self) -> QueueTuple: - ... - - def finish_response(self, result: Union[Response, Failure], request: Request) -> None: - ... - - def is_idle(self) -> bool: - ... - - def needs_backout(self) -> bool: - ... - - - -class Scraper: - def __init__(self, crawler: Crawler) -> None: - ... - - @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: - """Open the given spider for scraping and allocate resources for it""" - ... - - def close_spider(self, spider: Spider) -> Deferred: - """Close a spider being scraped and release its resources""" - ... - - def is_idle(self) -> bool: - """Return True if there isn't any more spiders to process""" - ... - - def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: - ... - - def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: - ... - - def handle_spider_error(self, _failure: Failure, request: Request, response: Union[Response, Failure], spider: Spider) -> None: - ... - - def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, response: Union[Response, Failure], spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/core/spidermw.pyi b/typings/scrapy/core/spidermw.pyi deleted file mode 100644 index dfd0ccf0..00000000 --- a/typings/scrapy/core/spidermw.pyi +++ /dev/null @@ -1,31 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Callable, Iterable, Union -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response -from scrapy.middleware import MiddlewareManager - -""" -Spider Middleware manager - -See documentation in docs/topics/spider-middleware.rst -""" -logger = ... -ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] -class SpiderMiddlewareManager(MiddlewareManager): - component_name = ... - def __init__(self, *middlewares: Any) -> None: - ... - - def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: - ... - - def process_start_requests(self, start_requests: Iterable[Request], spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/crawler.pyi b/typings/scrapy/crawler.pyi deleted file mode 100644 index b6f77ea0..00000000 --- a/typings/scrapy/crawler.pyi +++ /dev/null @@ -1,146 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Dict, Generator, TYPE_CHECKING, Type, Union -from twisted.internet.defer import Deferred, inlineCallbacks -from scrapy import Spider -from scrapy.settings import Settings - -if TYPE_CHECKING: - ... -logger = ... -class Crawler: - def __init__(self, spidercls: Type[Spider], settings: Union[None, Dict[str, Any], Settings] = ..., init_reactor: bool = ...) -> None: - ... - - @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: - ... - - @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: - """Starts a graceful stop of the crawler and returns a deferred that is - fired when the crawler is stopped.""" - ... - - - -class CrawlerRunner: - """ - This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup :mod:`~twisted.internet.reactor`. - - The CrawlerRunner object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - crawlers = ... - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = ...) -> None: - ... - - def crawl(self, crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any) -> Deferred: - """ - Run a crawler with the provided arguments. - - It will call the given Crawler's :meth:`~Crawler.crawl` method, while - keeping track of it so it can be stopped later. - - If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` - instance, this method will try to create one using this parameter as - the spider class given to it. - - Returns a deferred that is fired when the crawling is finished. - - :param crawler_or_spidercls: already created crawler, or a spider class - or spider's name inside the project to create it - :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, - :class:`~scrapy.spiders.Spider` subclass or string - - :param args: arguments to initialize the spider - - :param kwargs: keyword arguments to initialize the spider - """ - ... - - def create_crawler(self, crawler_or_spidercls: Union[Type[Spider], str, Crawler]) -> Crawler: - """ - Return a :class:`~scrapy.crawler.Crawler` object. - - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. - * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. - * If ``crawler_or_spidercls`` is a string, this function finds - a spider with this name in a Scrapy project (using spider loader), - then creates a Crawler instance for it. - """ - ... - - def stop(self) -> Deferred: - """ - Stops simultaneously all the crawling jobs taking place. - - Returns a deferred that is fired when they all have ended. - """ - ... - - @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: - """ - join() - - Returns a deferred that is fired when all managed :attr:`crawlers` have - completed their executions. - """ - ... - - - -class CrawlerProcess(CrawlerRunner): - """ - A class to run multiple scrapy crawlers in a process simultaneously. - - This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a :mod:`~twisted.internet.reactor` and handling shutdown - signals, like the keyboard interrupt command Ctrl-C. It also configures - top-level logging. - - This utility should be a better fit than - :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - :mod:`~twisted.internet.reactor` within your application. - - The CrawlerProcess object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - :param install_root_handler: whether to install root logging handler - (default: True) - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = ..., install_root_handler: bool = ...) -> None: - ... - - def start(self, stop_after_crawl: bool = ..., install_signal_handlers: bool = ...) -> None: - """ - This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool - size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache - based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. - - If ``stop_after_crawl`` is True, the reactor will be stopped after all - crawlers have finished, using :meth:`join`. - - :param bool stop_after_crawl: stop or not the reactor when all - crawlers have finished - - :param bool install_signal_handlers: whether to install the shutdown - handlers (default: True) - """ - ... - - - diff --git a/typings/scrapy/downloadermiddlewares/__init__.pyi b/typings/scrapy/downloadermiddlewares/__init__.pyi deleted file mode 100644 index 006bc274..00000000 --- a/typings/scrapy/downloadermiddlewares/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - diff --git a/typings/scrapy/dupefilters.pyi b/typings/scrapy/dupefilters.pyi deleted file mode 100644 index 9dbe7f6d..00000000 --- a/typings/scrapy/dupefilters.pyi +++ /dev/null @@ -1,62 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Optional, TYPE_CHECKING -from twisted.internet.defer import Deferred -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider -from scrapy.utils.request import RequestFingerprinterProtocol -from typing_extensions import Self -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -class BaseDupeFilter: - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - ... - - def request_seen(self, request: Request) -> bool: - ... - - def open(self) -> Optional[Deferred]: - ... - - def close(self, reason: str) -> Optional[Deferred]: - ... - - def log(self, request: Request, spider: Spider) -> None: - """Log that a request has been filtered""" - ... - - - -class RFPDupeFilter(BaseDupeFilter): - """Request Fingerprint duplicates filter""" - def __init__(self, path: Optional[str] = ..., debug: bool = ..., *, fingerprinter: Optional[RequestFingerprinterProtocol] = ...) -> None: - ... - - @classmethod - def from_settings(cls, settings: BaseSettings, *, fingerprinter: Optional[RequestFingerprinterProtocol] = ...) -> Self: - ... - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - ... - - def request_seen(self, request: Request) -> bool: - ... - - def request_fingerprint(self, request: Request) -> str: - ... - - def close(self, reason: str) -> None: - ... - - def log(self, request: Request, spider: Spider) -> None: - ... - - - diff --git a/typings/scrapy/exceptions.pyi b/typings/scrapy/exceptions.pyi deleted file mode 100644 index a9f486eb..00000000 --- a/typings/scrapy/exceptions.pyi +++ /dev/null @@ -1,82 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any - -""" -Scrapy core exceptions - -These exceptions are documented in docs/topics/exceptions.rst. Please don't add -new exceptions here without documenting them there. -""" -class NotConfigured(Exception): - """Indicates a missing configuration situation""" - ... - - -class _InvalidOutput(TypeError): - """ - Indicates an invalid value has been returned by a middleware's processing method. - Internal and undocumented, it should not be raised or caught by user code. - """ - ... - - -class IgnoreRequest(Exception): - """Indicates a decision was made not to process a request""" - ... - - -class DontCloseSpider(Exception): - """Request the spider not to be closed yet""" - ... - - -class CloseSpider(Exception): - """Raise this from callbacks to request the spider to be closed""" - def __init__(self, reason: str = ...) -> None: - ... - - - -class StopDownload(Exception): - """ - Stop the download of the body for a given response. - The 'fail' boolean parameter indicates whether or not the resulting partial response - should be handled by the request errback. Note that 'fail' is a keyword-only argument. - """ - def __init__(self, *, fail: bool = ...) -> None: - ... - - - -class DropItem(Exception): - """Drop item from the item pipeline""" - ... - - -class NotSupported(Exception): - """Indicates a feature or method is not supported""" - ... - - -class UsageError(Exception): - """To indicate a command-line usage error""" - def __init__(self, *a: Any, **kw: Any) -> None: - ... - - - -class ScrapyDeprecationWarning(Warning): - """Warning category for deprecated features, since the default - DeprecationWarning is silenced on Python 2.7+ - """ - ... - - -class ContractFail(AssertionError): - """Error raised in case of a failing contract""" - ... - - diff --git a/typings/scrapy/exporters.pyi b/typings/scrapy/exporters.pyi deleted file mode 100644 index 58131e62..00000000 --- a/typings/scrapy/exporters.pyi +++ /dev/null @@ -1,131 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Item Exporters are used to export/serialize items into different formats. -""" -__all__ = ["BaseItemExporter", "PprintItemExporter", "PickleItemExporter", "CsvItemExporter", "XmlItemExporter", "JsonLinesItemExporter", "JsonItemExporter", "MarshalItemExporter"] -class BaseItemExporter: - def __init__(self, *, dont_fail=..., **kwargs) -> None: - ... - - def export_item(self, item): - ... - - def serialize_field(self, field, name, value): - ... - - def start_exporting(self): # -> None: - ... - - def finish_exporting(self): # -> None: - ... - - - -class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs) -> None: - ... - - def export_item(self, item): # -> None: - ... - - - -class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs) -> None: - ... - - def start_exporting(self): # -> None: - ... - - def finish_exporting(self): # -> None: - ... - - def export_item(self, item): # -> None: - ... - - - -class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs) -> None: - ... - - def start_exporting(self): # -> None: - ... - - def export_item(self, item): # -> None: - ... - - def finish_exporting(self): # -> None: - ... - - - -class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=..., join_multivalued=..., errors=..., **kwargs) -> None: - ... - - def serialize_field(self, field, name, value): - ... - - def export_item(self, item): # -> None: - ... - - def finish_exporting(self): # -> None: - ... - - - -class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=..., **kwargs) -> None: - ... - - def export_item(self, item): # -> None: - ... - - - -class MarshalItemExporter(BaseItemExporter): - """Exports items in a Python-specific binary format (see - :mod:`marshal`). - - :param file: The file-like object to use for exporting the data. Its - ``write`` method should accept :class:`bytes` (a disk file - opened in binary mode, a :class:`~io.BytesIO` object, etc) - """ - def __init__(self, file, **kwargs) -> None: - ... - - def export_item(self, item): # -> None: - ... - - - -class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs) -> None: - ... - - def export_item(self, item): # -> None: - ... - - - -class PythonItemExporter(BaseItemExporter): - """This is a base class for item exporters that extends - :class:`BaseItemExporter` with support for nested items. - - It serializes items to built-in Python types, so that any serialization - library (e.g. :mod:`json` or msgpack_) can be used on top of it. - - .. _msgpack: https://pypi.org/project/msgpack/ - """ - def serialize_field(self, field, name, value): - ... - - def export_item(self, item): # -> dict[str | Any, Any | None]: - ... - - - diff --git a/typings/scrapy/extension.pyi b/typings/scrapy/extension.pyi deleted file mode 100644 index 2ed1f05b..00000000 --- a/typings/scrapy/extension.pyi +++ /dev/null @@ -1,15 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.middleware import MiddlewareManager - -""" -The Extension Manager - -See documentation in docs/topics/extensions.rst -""" -class ExtensionManager(MiddlewareManager): - component_name = ... - - diff --git a/typings/scrapy/extensions/__init__.pyi b/typings/scrapy/extensions/__init__.pyi deleted file mode 100644 index 006bc274..00000000 --- a/typings/scrapy/extensions/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - diff --git a/typings/scrapy/http/__init__.pyi b/typings/scrapy/http/__init__.pyi deleted file mode 100644 index 0209a618..00000000 --- a/typings/scrapy/http/__init__.pyi +++ /dev/null @@ -1,20 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.http.headers import Headers -from scrapy.http.request import Request -from scrapy.http.request.form import FormRequest -from scrapy.http.request.json_request import JsonRequest -from scrapy.http.request.rpc import XmlRpcRequest -from scrapy.http.response import Response -from scrapy.http.response.html import HtmlResponse -from scrapy.http.response.text import TextResponse -from scrapy.http.response.xml import XmlResponse - -""" -Module containing all HTTP related classes - -Use this module (instead of the more specific ones) when importing Headers, -Request and Response outside this module. -""" diff --git a/typings/scrapy/http/common.pyi b/typings/scrapy/http/common.pyi deleted file mode 100644 index 91e1e91e..00000000 --- a/typings/scrapy/http/common.pyi +++ /dev/null @@ -1,7 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -def obsolete_setter(setter, attrname): # -> Callable[..., NoReturn]: - ... - diff --git a/typings/scrapy/http/headers.pyi b/typings/scrapy/http/headers.pyi deleted file mode 100644 index adf19857..00000000 --- a/typings/scrapy/http/headers.pyi +++ /dev/null @@ -1,61 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.utils.datatypes import CaselessDict - -class Headers(CaselessDict): - """Case insensitive http headers dictionary""" - def __init__(self, seq=..., encoding=...) -> None: - ... - - def update(self, seq): # -> None: - ... - - def normkey(self, key): # -> bytes: - """Normalize key to bytes""" - ... - - def normvalue(self, value): # -> list[bytes]: - """Normalize values to bytes""" - ... - - def __getitem__(self, key): # -> None: - ... - - def get(self, key, def_val=...): # -> None: - ... - - def getlist(self, key, def_val=...): # -> list[bytes] | list[Any]: - ... - - def setlist(self, key, list_): # -> None: - ... - - def setlistdefault(self, key, default_list=...): - ... - - def appendlist(self, key, value): # -> None: - ... - - def items(self): # -> Generator[tuple[Any, Any | list[bytes] | list[Any]], None, None]: - ... - - def values(self): # -> list[Any | None]: - ... - - def to_string(self): # -> bytes | None: - ... - - def to_unicode_dict(self): # -> CaseInsensitiveDict: - """Return headers as a CaselessDict with unicode keys - and unicode values. Multiple values are joined with ','. - """ - ... - - def __copy__(self): # -> Self: - ... - - copy = ... - - diff --git a/typings/scrapy/http/request/__init__.pyi b/typings/scrapy/http/request/__init__.pyi deleted file mode 100644 index 5843bd31..00000000 --- a/typings/scrapy/http/request/__init__.pyi +++ /dev/null @@ -1,116 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import inspect -import scrapy -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union -from w3lib.url import safe_url_string -from scrapy.http.common import obsolete_setter -from scrapy.http.headers import Headers -from scrapy.utils.curl import curl_to_request_kwargs -from scrapy.utils.python import to_bytes -from scrapy.utils.trackref import object_ref -from scrapy.utils.url import escape_ajax - -""" -This module implements the Request class which is used to represent HTTP -requests in Scrapy. - -See documentation in docs/topics/request-response.rst -""" -RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -def NO_CALLBACK(*args, **kwargs): - """When assigned to the ``callback`` parameter of - :class:`~scrapy.http.Request`, it indicates that the request is not meant - to have a spider callback at all. - - For example: - - .. code-block:: python - - Request("https://example.com", callback=NO_CALLBACK) - - This value should be used by :ref:`components ` that - create and handle their own requests, e.g. through - :meth:`scrapy.core.engine.ExecutionEngine.download`, so that downloader - middlewares handling such requests can treat them differently from requests - intended for the :meth:`~scrapy.Spider.parse` callback. - """ - ... - -class Request(object_ref): - """Represents an HTTP request, which is usually generated in a Spider and - executed by the Downloader, thus generating a :class:`Response`. - """ - attributes: Tuple[str, ...] = ... - def __init__(self, url: str, callback: Optional[Callable] = ..., method: str = ..., headers: Optional[dict] = ..., body: Optional[Union[bytes, str]] = ..., cookies: Optional[Union[dict, List[dict]]] = ..., meta: Optional[dict] = ..., encoding: str = ..., priority: int = ..., dont_filter: bool = ..., errback: Optional[Callable] = ..., flags: Optional[List[str]] = ..., cb_kwargs: Optional[dict] = ...) -> None: - ... - - @property - def cb_kwargs(self) -> dict: - ... - - @property - def meta(self) -> dict: - ... - - url = ... - body = ... - @property - def encoding(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def copy(self) -> Request: - ... - - def replace(self, *args, **kwargs) -> Request: - """Create a new Request with the same attributes except for those given new values""" - ... - - @classmethod - def from_curl(cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = ..., **kwargs) -> RequestTypeVar: - """Create a Request object from a string containing a `cURL - `_ command. It populates the HTTP method, the - URL, the headers, the cookies and the body. It accepts the same - arguments as the :class:`Request` class, taking preference and - overriding the values of the same arguments contained in the cURL - command. - - Unrecognized options are ignored by default. To raise an error when - finding unknown options call this method by passing - ``ignore_unknown_options=False``. - - .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` - subclasses, such as :class:`~scrapy.http.JSONRequest`, or - :class:`~scrapy.http.XmlRpcRequest`, as well as having - :ref:`downloader middlewares ` - and - :ref:`spider middlewares ` - enabled, such as - :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`, - :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, - or - :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, - may modify the :class:`~scrapy.http.Request` object. - - To translate a cURL command into a Scrapy request, - you may use `curl2scrapy `_. - """ - ... - - def to_dict(self, *, spider: Optional[scrapy.Spider] = ...) -> dict: - """Return a dictionary containing the Request's data. - - Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. - - If a spider is given, this method will try to find out the name of the spider methods used as callback - and errback and include them in the output dict, raising an exception if they cannot be found. - """ - ... - - - diff --git a/typings/scrapy/http/request/form.pyi b/typings/scrapy/http/request/form.pyi deleted file mode 100644 index b4b45923..00000000 --- a/typings/scrapy/http/request/form.pyi +++ /dev/null @@ -1,28 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union -from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse - -""" -This module implements the FormRequest class which is a more convenient class -(than Request) to generate Requests based on form data. - -See documentation in docs/topics/request-response.rst -""" -FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") -FormdataKVType = Tuple[str, Union[str, Iterable[str]]] -FormdataType = Optional[Union[dict, List[FormdataKVType]]] -class FormRequest(Request): - valid_form_methods = ... - def __init__(self, *args, formdata: FormdataType = ..., **kwargs) -> None: - ... - - @classmethod - def from_response(cls: Type[FormRequestTypeVar], response: TextResponse, formname: Optional[str] = ..., formid: Optional[str] = ..., formnumber: int = ..., formdata: FormdataType = ..., clickdata: Optional[dict] = ..., dont_click: bool = ..., formxpath: Optional[str] = ..., formcss: Optional[str] = ..., **kwargs) -> FormRequestTypeVar: - ... - - - diff --git a/typings/scrapy/http/request/json_request.pyi b/typings/scrapy/http/request/json_request.pyi deleted file mode 100644 index 7f912bac..00000000 --- a/typings/scrapy/http/request/json_request.pyi +++ /dev/null @@ -1,28 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Optional, Tuple -from scrapy.http.request import Request - -""" -This module implements the JsonRequest class which is a more convenient class -(than Request) to generate JSON Requests. - -See documentation in docs/topics/request-response.rst -""" -class JsonRequest(Request): - attributes: Tuple[str, ...] = ... - def __init__(self, *args, dumps_kwargs: Optional[dict] = ..., **kwargs) -> None: - ... - - @property - def dumps_kwargs(self) -> dict: - ... - - def replace(self, *args, **kwargs) -> Request: - ... - - - -JSONRequest = ... diff --git a/typings/scrapy/http/request/rpc.pyi b/typings/scrapy/http/request/rpc.pyi deleted file mode 100644 index 9cf84260..00000000 --- a/typings/scrapy/http/request/rpc.pyi +++ /dev/null @@ -1,20 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Optional -from scrapy.http.request import Request - -""" -This module implements the XmlRpcRequest class which is a more convenient class -(that Request) to generate xml-rpc requests. - -See documentation in docs/topics/request-response.rst -""" -DUMPS_ARGS = ... -class XmlRpcRequest(Request): - def __init__(self, *args, encoding: Optional[str] = ..., **kwargs) -> None: - ... - - - diff --git a/typings/scrapy/http/response/__init__.pyi b/typings/scrapy/http/response/__init__.pyi deleted file mode 100644 index d6106d35..00000000 --- a/typings/scrapy/http/response/__init__.pyi +++ /dev/null @@ -1,111 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Generator, Tuple -from urllib.parse import urljoin -from scrapy.exceptions import NotSupported -from scrapy.http.common import obsolete_setter -from scrapy.http.headers import Headers -from scrapy.http.request import Request -from scrapy.link import Link -from scrapy.utils.trackref import object_ref - -""" -This module implements the Response class which is used to represent HTTP -responses in Scrapy. - -See documentation in docs/topics/request-response.rst -""" -class Response(object_ref): - """An object that represents an HTTP response, which is usually - downloaded (by the Downloader) and fed to the Spiders for processing. - """ - attributes: Tuple[str, ...] = ... - def __init__(self, url: str, status=..., headers=..., body=..., flags=..., request=..., certificate=..., ip_address=..., protocol=...) -> None: - ... - - @property - def cb_kwargs(self): - ... - - @property - def meta(self): - ... - - url = ... - body = ... - def __repr__(self): # -> str: - ... - - def copy(self): - """Return a copy of this Response""" - ... - - def replace(self, *args, **kwargs): - """Create a new Response with the same attributes except for those given new values""" - ... - - def urljoin(self, url): # -> Any: - """Join this Response's url with a possible relative url to form an - absolute interpretation of the latter.""" - ... - - @property - def text(self): - """For subclasses of TextResponse, this will return the body - as str - """ - ... - - def css(self, *a, **kw): - """Shortcut method implemented only by responses whose content - is text (subclasses of TextResponse). - """ - ... - - def jmespath(self, *a, **kw): - """Shortcut method implemented only by responses whose content - is text (subclasses of TextResponse). - """ - ... - - def xpath(self, *a, **kw): - """Shortcut method implemented only by responses whose content - is text (subclasses of TextResponse). - """ - ... - - def follow(self, url, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Request: - """ - Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, - but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, - not only an absolute URL. - - :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` - method which supports selectors in addition to absolute/relative URLs - and Link objects. - - .. versionadded:: 2.0 - The *flags* parameter. - """ - ... - - def follow_all(self, urls, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Generator[Request, None, None]: - """ - .. versionadded:: 2.0 - - Return an iterable of :class:`~.Request` instances to follow all links - in ``urls``. It accepts the same arguments as ``Request.__init__`` method, - but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, - not only absolute URLs. - - :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all` - method which supports selectors in addition to absolute/relative URLs - and Link objects. - """ - ... - - - diff --git a/typings/scrapy/http/response/html.pyi b/typings/scrapy/http/response/html.pyi deleted file mode 100644 index dc9e6637..00000000 --- a/typings/scrapy/http/response/html.pyi +++ /dev/null @@ -1,16 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.http.response.text import TextResponse - -""" -This module implements the HtmlResponse class which adds encoding -discovering through HTML encoding declarations to the TextResponse class. - -See documentation in docs/topics/request-response.rst -""" -class HtmlResponse(TextResponse): - ... - - diff --git a/typings/scrapy/http/response/text.pyi b/typings/scrapy/http/response/text.pyi deleted file mode 100644 index 3f23e40c..00000000 --- a/typings/scrapy/http/response/text.pyi +++ /dev/null @@ -1,113 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Generator, TYPE_CHECKING, Tuple -from scrapy.http import Request -from scrapy.http.response import Response - -""" -This module implements the TextResponse class which adds encoding handling and -discovering (through HTTP headers) to base Response class. - -See documentation in docs/topics/request-response.rst -""" -if TYPE_CHECKING: - ... -_NONE = ... -class TextResponse(Response): - _DEFAULT_ENCODING = ... - _cached_decoded_json = ... - attributes: Tuple[str, ...] = ... - def __init__(self, *args: Any, **kwargs: Any) -> None: - ... - - @property - def encoding(self): # -> Any | str: - ... - - def json(self): # -> Any | object: - """ - .. versionadded:: 2.2 - - Deserialize a JSON document to a Python object. - """ - ... - - @property - def text(self) -> str: - """Body as unicode""" - ... - - def urljoin(self, url): # -> str: - """Join this Response's url with a possible relative url to form an - absolute interpretation of the latter.""" - ... - - @property - def selector(self): # -> Selector: - ... - - def jmespath(self, query, **kwargs): # -> SelectorList[Selector]: - ... - - def xpath(self, query, **kwargs): # -> SelectorList[Selector]: - ... - - def css(self, query): # -> SelectorList[Selector]: - ... - - def follow(self, url, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Request: - """ - Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, - but ``url`` can be not only an absolute URL, but also - - * a relative URL - * a :class:`~scrapy.link.Link` object, e.g. the result of - :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. - ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. - ``response.css('a::attr(href)')[0]`` or - ``response.xpath('//img/@src')[0]`` - - See :ref:`response-follow-example` for usage examples. - """ - ... - - def follow_all(self, urls=..., callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=..., css=..., xpath=...) -> Generator[Request, None, None]: - """ - A generator that produces :class:`~.Request` instances to follow all - links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s - ``__init__`` method, except that each ``urls`` element does not need to be - an absolute URL, it can be any of the following: - - * a relative URL - * a :class:`~scrapy.link.Link` object, e.g. the result of - :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. - ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. - ``response.css('a::attr(href)')[0]`` or - ``response.xpath('//img/@src')[0]`` - - In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction - within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). - - Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or - using the ``css`` or ``xpath`` parameters, this method will not produce requests for - selectors from which links cannot be obtained (for instance, anchor tags without an - ``href`` attribute) - """ - ... - - - -class _InvalidSelector(ValueError): - """ - Raised when a URL cannot be obtained from a Selector - """ - ... - - diff --git a/typings/scrapy/http/response/xml.pyi b/typings/scrapy/http/response/xml.pyi deleted file mode 100644 index 27a3928e..00000000 --- a/typings/scrapy/http/response/xml.pyi +++ /dev/null @@ -1,16 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.http.response.text import TextResponse - -""" -This module implements the XmlResponse class which adds encoding -discovering through XML encoding declarations to the TextResponse class. - -See documentation in docs/topics/request-response.rst -""" -class XmlResponse(TextResponse): - ... - - diff --git a/typings/scrapy/interfaces.pyi b/typings/scrapy/interfaces.pyi deleted file mode 100644 index e42719de..00000000 --- a/typings/scrapy/interfaces.pyi +++ /dev/null @@ -1,27 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from zope.interface import Interface - -class ISpiderLoader(Interface): - def from_settings(settings): # -> None: - """Return an instance of the class for the given settings""" - ... - - def load(spider_name): # -> None: - """Return the Spider class for the given spider name. If the spider - name is not found, it must raise a KeyError.""" - ... - - def list(): # -> None: - """Return a list with the names of all spiders available in the - project""" - ... - - def find_by_request(request): # -> None: - """Return the list of spiders names that can handle the given request""" - ... - - - diff --git a/typings/scrapy/item.pyi b/typings/scrapy/item.pyi deleted file mode 100644 index adeacc8e..00000000 --- a/typings/scrapy/item.pyi +++ /dev/null @@ -1,92 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from abc import ABCMeta -from collections.abc import MutableMapping -from typing import Dict -from scrapy.utils.trackref import object_ref - -""" -Scrapy Item - -See documentation in docs/topics/item.rst -""" -class Field(dict): - """Container of field metadata""" - ... - - -class ItemMeta(ABCMeta): - """Metaclass_ of :class:`Item` that handles field definitions. - - .. _metaclass: https://realpython.com/python-metaclasses - """ - def __new__(mcs, class_name, bases, attrs): # -> Self: - ... - - - -class Item(MutableMapping, object_ref, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ - fields: Dict[str, Field] - def __init__(self, *args, **kwargs) -> None: - ... - - def __getitem__(self, key): - ... - - def __setitem__(self, key, value): # -> None: - ... - - def __delitem__(self, key): # -> None: - ... - - def __getattr__(self, name): - ... - - def __setattr__(self, name, value): # -> None: - ... - - def __len__(self): # -> int: - ... - - def __iter__(self): # -> Iterator[Any]: - ... - - __hash__ = ... - def keys(self): # -> dict_keys[Any, Any]: - ... - - def __repr__(self): # -> str: - ... - - def copy(self): # -> Self: - ... - - def deepcopy(self): # -> Self: - """Return a :func:`~copy.deepcopy` of this item.""" - ... - - - diff --git a/typings/scrapy/link.pyi b/typings/scrapy/link.pyi deleted file mode 100644 index 8b69c2e1..00000000 --- a/typings/scrapy/link.pyi +++ /dev/null @@ -1,44 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any - -""" -This module defines the Link object used in Link extractors. - -For actual link extractors implementation see scrapy.linkextractors, or -its documentation in: docs/topics/link-extractors.rst -""" -class Link: - """Link objects represent an extracted link by the LinkExtractor. - - Using the anchor tag sample below to illustrate the parameters:: - - Dont follow this one - - :param url: the absolute url being linked to in the anchor tag. - From the sample, this is ``https://example.com/nofollow.html``. - - :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``. - - :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``. - - :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute - of the anchor tag. - """ - __slots__ = ... - def __init__(self, url: str, text: str = ..., fragment: str = ..., nofollow: bool = ...) -> None: - ... - - def __eq__(self, other: Any) -> bool: - ... - - def __hash__(self) -> int: - ... - - def __repr__(self) -> str: - ... - - - diff --git a/typings/scrapy/linkextractors/__init__.pyi b/typings/scrapy/linkextractors/__init__.pyi deleted file mode 100644 index 79b49626..00000000 --- a/typings/scrapy/linkextractors/__init__.pyi +++ /dev/null @@ -1,16 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import re -from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor - -""" -scrapy.linkextractors - -This package contains a collection of Link Extractors. - -For more info see docs/topics/link-extractors.rst -""" -IGNORED_EXTENSIONS = ... -_re_type = ... diff --git a/typings/scrapy/linkextractors/lxmlhtml.pyi b/typings/scrapy/linkextractors/lxmlhtml.pyi deleted file mode 100644 index d8979d82..00000000 --- a/typings/scrapy/linkextractors/lxmlhtml.pyi +++ /dev/null @@ -1,41 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Link extractor based on lxml.html -""" -logger = ... -XHTML_NAMESPACE = ... -_collect_string_content = ... -class LxmlParserLinkExtractor: - def __init__(self, tag=..., attr=..., process=..., unique=..., strip=..., canonicalized=...) -> None: - ... - - def extract_links(self, response): # -> list[Any]: - ... - - - -class LxmlLinkExtractor: - _csstranslator = ... - def __init__(self, allow=..., deny=..., allow_domains=..., deny_domains=..., restrict_xpaths=..., tags=..., attrs=..., canonicalize=..., unique=..., process_value=..., deny_extensions=..., restrict_css=..., strip=..., restrict_text=...) -> None: - ... - - def matches(self, url): # -> bool: - ... - - def extract_links(self, response): # -> list[Any]: - """Returns a list of :class:`~scrapy.link.Link` objects from the - specified :class:`response `. - - Only links that match the settings passed to the ``__init__`` method of - the link extractor are returned. - - Duplicate links are omitted if the ``unique`` attribute is set to ``True``, - otherwise they are returned. - """ - ... - - - diff --git a/typings/scrapy/loader/__init__.pyi b/typings/scrapy/loader/__init__.pyi deleted file mode 100644 index be8853cd..00000000 --- a/typings/scrapy/loader/__init__.pyi +++ /dev/null @@ -1,88 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import itemloaders -from scrapy.item import Item -from scrapy.selector import Selector - -""" -Item Loader - -See documentation in docs/topics/loaders.rst -""" -class ItemLoader(itemloaders.ItemLoader): - """ - A user-friendly abstraction to populate an :ref:`item ` with data - by applying :ref:`field processors ` to scraped data. - When instantiated with a ``selector`` or a ``response`` it supports - data extraction from web pages using :ref:`selectors `. - - :param item: The item instance to populate using subsequent calls to - :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, - or :meth:`~ItemLoader.add_value`. - :type item: scrapy.item.Item - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or - :meth:`replace_css` method. - :type selector: :class:`~scrapy.selector.Selector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - If no item is given, one is instantiated automatically using the class in - :attr:`default_item_class`. - - The item, selector, response and remaining keyword arguments are - assigned to the Loader context (accessible through the :attr:`context` attribute). - - .. attribute:: item - - The item object being parsed by this Item Loader. - This is mostly used as a property so, when attempting to override this - value, you may want to check out :attr:`default_item_class` first. - - .. attribute:: context - - The currently active :ref:`Context ` of this Item Loader. - - .. attribute:: default_item_class - - An :ref:`item ` class (or factory), used to instantiate - items when not given in the ``__init__`` method. - - .. attribute:: default_input_processor - - The default input processor to use for those fields which don't specify - one. - - .. attribute:: default_output_processor - - The default output processor to use for those fields which don't specify - one. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the ``__init__`` method. - If a selector is given in the ``__init__`` method this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the ``__init__`` method or one created from - the response given in the ``__init__`` method using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. - """ - default_item_class = Item - default_selector_class = Selector - def __init__(self, item=..., selector=..., response=..., parent=..., **context) -> None: - ... - - - diff --git a/typings/scrapy/logformatter.pyi b/typings/scrapy/logformatter.pyi deleted file mode 100644 index 94f15400..00000000 --- a/typings/scrapy/logformatter.pyi +++ /dev/null @@ -1,98 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Optional, TYPE_CHECKING, Union -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response -from typing_extensions import Self -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -SCRAPEDMSG = ... -DROPPEDMSG = ... -CRAWLEDMSG = ... -ITEMERRORMSG = ... -SPIDERERRORMSG = ... -DOWNLOADERRORMSG_SHORT = ... -DOWNLOADERRORMSG_LONG = ... -class LogFormatter: - """Class for generating log messages for different actions. - - All methods must return a dictionary listing the parameters ``level``, ``msg`` - and ``args`` which are going to be used for constructing the log message when - calling ``logging.log``. - - Dictionary keys for the method outputs: - - * ``level`` is the log level for that action, you can use those from the - `python logging library `_ : - ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` - and ``logging.CRITICAL``. - * ``msg`` should be a string that can contain different formatting placeholders. - This string, formatted with the provided ``args``, is going to be the long message - for that action. - * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. - The final log message is computed as ``msg % args``. - - Users can define their own ``LogFormatter`` class if they want to customize how - each action is logged or if they want to omit it entirely. In order to omit - logging an action the method must return ``None``. - - Here is an example on how to create a custom log formatter to lower the severity level of - the log message when an item is dropped from the pipeline:: - - class PoliteLogFormatter(logformatter.LogFormatter): - def dropped(self, item, exception, response, spider): - return { - 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", - 'args': { - 'exception': exception, - 'item': item, - } - } - """ - def crawled(self, request: Request, response: Response, spider: Spider) -> dict: - """Logs a message when the crawler finds a webpage.""" - ... - - def scraped(self, item: Any, response: Union[Response, Failure], spider: Spider) -> dict: - """Logs a message when an item is scraped by a spider.""" - ... - - def dropped(self, item: Any, exception: BaseException, response: Response, spider: Spider) -> dict: - """Logs a message when an item is dropped while it is passing through the item pipeline.""" - ... - - def item_error(self, item: Any, exception: BaseException, response: Response, spider: Spider) -> dict: - """Logs a message when an item causes an error while it is passing - through the item pipeline. - - .. versionadded:: 2.0 - """ - ... - - def spider_error(self, failure: Failure, request: Request, response: Union[Response, Failure], spider: Spider) -> dict: - """Logs an error message from a spider. - - .. versionadded:: 2.0 - """ - ... - - def download_error(self, failure: Failure, request: Request, spider: Spider, errmsg: Optional[str] = ...) -> dict: - """Logs a download error message from a spider (typically coming from - the engine). - - .. versionadded:: 2.0 - """ - ... - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - ... - - - diff --git a/typings/scrapy/mail.pyi b/typings/scrapy/mail.pyi deleted file mode 100644 index dcfd80f2..00000000 --- a/typings/scrapy/mail.pyi +++ /dev/null @@ -1,24 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Mail sending helpers - -See documentation in docs/topics/email.rst -""" -logger = ... -COMMASPACE = ... -class MailSender: - def __init__(self, smtphost=..., mailfrom=..., smtpuser=..., smtppass=..., smtpport=..., smtptls=..., smtpssl=..., debug=...) -> None: - ... - - @classmethod - def from_settings(cls, settings): # -> Self: - ... - - def send(self, to, subject, body, cc=..., attachs=..., mimetype=..., charset=..., _callback=...): # -> Deferred[Any] | None: - ... - - - diff --git a/typings/scrapy/middleware.pyi b/typings/scrapy/middleware.pyi deleted file mode 100644 index f7c23a20..00000000 --- a/typings/scrapy/middleware.pyi +++ /dev/null @@ -1,36 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Optional, TYPE_CHECKING -from twisted.internet.defer import Deferred -from scrapy import Spider -from scrapy.settings import Settings -from typing_extensions import Self -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -logger = ... -class MiddlewareManager: - """Base class for implementing middleware managers""" - component_name = ... - def __init__(self, *middlewares: Any) -> None: - ... - - @classmethod - def from_settings(cls, settings: Settings, crawler: Optional[Crawler] = ...) -> Self: - ... - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - ... - - def open_spider(self, spider: Spider) -> Deferred: - ... - - def close_spider(self, spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/pipelines/__init__.pyi b/typings/scrapy/pipelines/__init__.pyi deleted file mode 100644 index 78efa43a..00000000 --- a/typings/scrapy/pipelines/__init__.pyi +++ /dev/null @@ -1,23 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, List -from twisted.internet.defer import Deferred -from scrapy import Spider -from scrapy.middleware import MiddlewareManager -from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import deferred_f_from_coro_f - -""" -Item pipeline - -See documentation in docs/item-pipeline.rst -""" -class ItemPipelineManager(MiddlewareManager): - component_name = ... - def process_item(self, item: Any, spider: Spider) -> Deferred: - ... - - - diff --git a/typings/scrapy/pqueues.pyi b/typings/scrapy/pqueues.pyi deleted file mode 100644 index b0918e7a..00000000 --- a/typings/scrapy/pqueues.pyi +++ /dev/null @@ -1,122 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -logger = ... -class ScrapyPriorityQueue: - """A priority queue implemented using multiple internal queues (typically, - FIFO queues). It uses one internal queue for each priority value. The internal - queue must implement the following methods: - - * push(obj) - * pop() - * close() - * __len__() - - Optionally, the queue could provide a ``peek`` method, that should return the - next object to be returned by ``pop``, but without removing it from the queue. - - ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls - argument, which is a class used to instantiate a new (internal) queue when - a new priority is allocated. - - Only integer priorities should be used. Lower numbers are higher - priorities. - - startprios is a sequence of priorities to start with. If the queue was - previously closed leaving some priority buckets non-empty, those priorities - should be passed in startprios. - - """ - @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=...): # -> Self: - ... - - def __init__(self, crawler, downstream_queue_cls, key, startprios=...) -> None: - ... - - def init_prios(self, startprios): # -> None: - ... - - def qfactory(self, key): - ... - - def priority(self, request): - ... - - def push(self, request): # -> None: - ... - - def pop(self): # -> None: - ... - - def peek(self): # -> None: - """Returns the next object to be returned by :meth:`pop`, - but without removing it from the queue. - - Raises :exc:`NotImplementedError` if the underlying queue class does - not implement a ``peek`` method, which is optional for queues. - """ - ... - - def close(self): # -> list[Any]: - ... - - def __len__(self): # -> int: - ... - - - -class DownloaderInterface: - def __init__(self, crawler) -> None: - ... - - def stats(self, possible_slots): # -> list[tuple[int, Any]]: - ... - - def get_slot_key(self, request): - ... - - - -class DownloaderAwarePriorityQueue: - """PriorityQueue which takes Downloader activity into account: - domains (slots) with the least amount of active downloads are dequeued - first. - """ - @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=...): # -> Self: - ... - - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=...) -> None: - ... - - def pqfactory(self, slot, startprios=...): # -> ScrapyPriorityQueue: - ... - - def pop(self): # -> None: - ... - - def push(self, request): # -> None: - ... - - def peek(self): # -> None: - """Returns the next object to be returned by :meth:`pop`, - but without removing it from the queue. - - Raises :exc:`NotImplementedError` if the underlying queue class does - not implement a ``peek`` method, which is optional for queues. - """ - ... - - def close(self): # -> dict[Any, Any]: - ... - - def __len__(self): # -> int: - ... - - def __contains__(self, slot): # -> bool: - ... - - - diff --git a/typings/scrapy/resolver.pyi b/typings/scrapy/resolver.pyi deleted file mode 100644 index ba235dad..00000000 --- a/typings/scrapy/resolver.pyi +++ /dev/null @@ -1,78 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any -from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple -from zope.interface.declarations import implementer, provider -from scrapy.utils.datatypes import LocalCache - -dnscache: LocalCache[str, Any] = ... -@implementer(IResolverSimple) -class CachingThreadedResolver(ThreadedResolver): - """ - Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. - """ - def __init__(self, reactor, cache_size, timeout) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, reactor): # -> Self: - ... - - def install_on_reactor(self): # -> None: - ... - - def getHostByName(self, name: str, timeout=...): # -> Deferred[Any] | Deferred[str]: - ... - - - -@implementer(IHostResolution) -class HostResolution: - def __init__(self, name) -> None: - ... - - def cancel(self): - ... - - - -@provider(IResolutionReceiver) -class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName) -> None: - ... - - def resolutionBegan(self, resolution): # -> None: - ... - - def addressResolved(self, address): # -> None: - ... - - def resolutionComplete(self): # -> None: - ... - - - -@implementer(IHostnameResolver) -class CachingHostnameResolver: - """ - Experimental caching resolver. Resolves IPv4 and IPv6 addresses, - does not support setting a timeout value for DNS requests. - """ - def __init__(self, reactor, cache_size) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, reactor): # -> Self: - ... - - def install_on_reactor(self): # -> None: - ... - - def resolveHostName(self, resolutionReceiver, hostName: str, portNumber=..., addressTypes=..., transportSemantics=...): - ... - - - diff --git a/typings/scrapy/responsetypes.pyi b/typings/scrapy/responsetypes.pyi deleted file mode 100644 index 3e13c24f..00000000 --- a/typings/scrapy/responsetypes.pyi +++ /dev/null @@ -1,52 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Mapping, Optional, Type, Union -from scrapy.http import Response - -""" -This module implements a class which returns the appropriate Response class -based on different criteria. -""" -class ResponseTypes: - CLASSES = ... - def __init__(self) -> None: - ... - - def from_mimetype(self, mimetype: str) -> Type[Response]: - """Return the most appropriate Response class for the given mimetype""" - ... - - def from_content_type(self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = ...) -> Type[Response]: - """Return the most appropriate Response class from an HTTP Content-Type - header""" - ... - - def from_content_disposition(self, content_disposition: Union[str, bytes]) -> Type[Response]: - ... - - def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: - """Return the most appropriate Response class by looking at the HTTP - headers""" - ... - - def from_filename(self, filename: str) -> Type[Response]: - """Return the most appropriate Response class from a file name""" - ... - - def from_body(self, body: bytes) -> Type[Response]: - """Try to guess the appropriate response based on the body content. - This method is a bit magic and could be improved in the future, but - it's not meant to be used except for special cases where response types - cannot be guess using more straightforward methods.""" - ... - - def from_args(self, headers: Optional[Mapping[bytes, bytes]] = ..., url: Optional[str] = ..., filename: Optional[str] = ..., body: Optional[bytes] = ...) -> Type[Response]: - """Guess the most appropriate Response class based on - the given arguments.""" - ... - - - -responsetypes = ... diff --git a/typings/scrapy/robotstxt.pyi b/typings/scrapy/robotstxt.pyi deleted file mode 100644 index 28f28d02..00000000 --- a/typings/scrapy/robotstxt.pyi +++ /dev/null @@ -1,91 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from abc import ABCMeta, abstractmethod - -logger = ... -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=...): # -> str: - ... - -class RobotParser(metaclass=ABCMeta): - @classmethod - @abstractmethod - def from_crawler(cls, crawler, robotstxt_body): # -> None: - """Parse the content of a robots.txt_ file as bytes. This must be a class method. - It must return a new instance of the parser backend. - - :param crawler: crawler which made the request - :type crawler: :class:`~scrapy.crawler.Crawler` instance - - :param robotstxt_body: content of a robots.txt_ file. - :type robotstxt_body: bytes - """ - ... - - @abstractmethod - def allowed(self, url, user_agent): # -> None: - """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. - - :param url: Absolute URL - :type url: str - - :param user_agent: User agent - :type user_agent: str - """ - ... - - - -class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, robotstxt_body): # -> Self: - ... - - def allowed(self, url, user_agent): # -> bool: - ... - - - -class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, robotstxt_body): # -> Self: - ... - - def allowed(self, url, user_agent): - ... - - - -class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, robotstxt_body): # -> Self: - ... - - def allowed(self, url, user_agent): - ... - - - -class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider) -> None: - ... - - @classmethod - def from_crawler(cls, crawler, robotstxt_body): # -> Self: - ... - - def allowed(self, url, user_agent): # -> Literal[True]: - ... - - - diff --git a/typings/scrapy/selector/__init__.pyi b/typings/scrapy/selector/__init__.pyi deleted file mode 100644 index ae51a54a..00000000 --- a/typings/scrapy/selector/__init__.pyi +++ /dev/null @@ -1,9 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.selector.unified import Selector, SelectorList - -""" -Selectors -""" diff --git a/typings/scrapy/selector/unified.pyi b/typings/scrapy/selector/unified.pyi deleted file mode 100644 index e329f020..00000000 --- a/typings/scrapy/selector/unified.pyi +++ /dev/null @@ -1,59 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Optional -from parsel import Selector as _ParselSelector -from scrapy.http import TextResponse -from scrapy.utils.trackref import object_ref - -""" -XPath selectors based on lxml -""" -__all__ = ["Selector", "SelectorList"] -_NOT_SET = ... -class SelectorList(_ParselSelector.selectorlist_cls, object_ref): - """ - The :class:`SelectorList` class is a subclass of the builtin ``list`` - class, which provides a few additional methods. - """ - ... - - -class Selector(_ParselSelector, object_ref): - """ - An instance of :class:`Selector` is a wrapper over response to select - certain parts of its content. - - ``response`` is an :class:`~scrapy.http.HtmlResponse` or an - :class:`~scrapy.http.XmlResponse` object that will be used for selecting - and extracting data. - - ``text`` is a unicode string or utf-8 encoded text for cases when a - ``response`` isn't available. Using ``text`` and ``response`` together is - undefined behavior. - - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` - or ``None`` (default). - - If ``type`` is ``None``, the selector automatically chooses the best type - based on ``response`` type (see below), or defaults to ``"html"`` in case it - is used together with ``text``. - - If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follows: - - * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type - * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type - * ``"html"`` for anything else - - Otherwise, if ``type`` is set, the selector type will be forced and no - detection will occur. - """ - __slots__ = ... - selectorlist_cls = SelectorList - def __init__(self, response: Optional[TextResponse] = ..., text: Optional[str] = ..., type: Optional[str] = ..., root: Optional[Any] = ..., **kwargs: Any) -> None: - ... - - - diff --git a/typings/scrapy/settings/__init__.pyi b/typings/scrapy/settings/__init__.pyi deleted file mode 100644 index 84fc9f45..00000000 --- a/typings/scrapy/settings/__init__.pyi +++ /dev/null @@ -1,371 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import copy -import json -from __future__ import annotations -from importlib import import_module -from pprint import pformat -from types import ModuleType -from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, TYPE_CHECKING, Tuple, Union, cast -from scrapy.settings import default_settings -from _typeshed import SupportsItems -from typing_extensions import Self - -_SettingsKeyT = Union[bool, float, int, str, None] -if TYPE_CHECKING: - _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] -SETTINGS_PRIORITIES: Dict[str, int] = ... -def get_settings_priority(priority: Union[int, str]) -> int: - """ - Small helper function that looks up a given string priority in the - :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its - numerical value, or directly returns a given numerical priority. - """ - ... - -class SettingsAttribute: - """Class for storing data related to settings attributes. - - This class is intended for internal usage, you should try Settings class - for settings configuration, not this one. - """ - def __init__(self, value: Any, priority: int) -> None: - ... - - def set(self, value: Any, priority: int) -> None: - """Sets value if priority is higher or equal than current priority.""" - ... - - def __repr__(self) -> str: - ... - - - -class BaseSettings(MutableMapping[_SettingsKeyT, Any]): - """ - Instances of this class behave like dictionaries, but store priorities - along with their ``(key, value)`` pairs, and can be frozen (i.e. marked - immutable). - - Key-value entries can be passed on initialization with the ``values`` - argument, and they would take the ``priority`` level (unless ``values`` is - already an instance of :class:`~scrapy.settings.BaseSettings`, in which - case the existing priority levels will be kept). If the ``priority`` - argument is a string, the priority name will be looked up in - :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer - should be provided. - - Once the object is created, new settings can be loaded or updated with the - :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with - the square bracket notation of dictionaries, or with the - :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its - value conversion variants. When requesting a stored key, the value with the - highest priority will be retrieved. - """ - __default = ... - def __init__(self, values: _SettingsInputT = ..., priority: Union[int, str] = ...) -> None: - ... - - def __getitem__(self, opt_name: _SettingsKeyT) -> Any: - ... - - def __contains__(self, name: Any) -> bool: - ... - - def get(self, name: _SettingsKeyT, default: Any = ...) -> Any: - """ - Get a setting value without affecting its original type. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getbool(self, name: _SettingsKeyT, default: bool = ...) -> bool: - """ - Get a setting value as a boolean. - - ``1``, ``'1'``, `True`` and ``'True'`` return ``True``, - while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``. - - For example, settings populated through environment variables set to - ``'0'`` will return ``False`` when using this method. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getint(self, name: _SettingsKeyT, default: int = ...) -> int: - """ - Get a setting value as an int. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getfloat(self, name: _SettingsKeyT, default: float = ...) -> float: - """ - Get a setting value as a float. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getlist(self, name: _SettingsKeyT, default: Optional[List[Any]] = ...) -> List[Any]: - """ - Get a setting value as a list. If the setting original type is a list, a - copy of it will be returned. If it's a string it will be split by ",". - - For example, settings populated through environment variables set to - ``'one,two'`` will return a list ['one', 'two'] when using this method. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getdict(self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = ...) -> Dict[Any, Any]: - """ - Get a setting value as a dictionary. If the setting original type is a - dictionary, a copy of it will be returned. If it is a string it will be - evaluated as a JSON dictionary. In the case that it is a - :class:`~scrapy.settings.BaseSettings` instance itself, it will be - converted to a dictionary, containing all its current settings values - as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`, - and losing all information about priority and mutability. - - :param name: the setting name - :type name: str - - :param default: the value to return if no setting is found - :type default: object - """ - ... - - def getdictorlist(self, name: _SettingsKeyT, default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = ...) -> Union[Dict[Any, Any], List[Any]]: - """Get a setting value as either a :class:`dict` or a :class:`list`. - - If the setting is already a dict or a list, a copy of it will be - returned. - - If it is a string it will be evaluated as JSON, or as a comma-separated - list of strings as a fallback. - - For example, settings populated from the command line will return: - - - ``{'key1': 'value1', 'key2': 'value2'}`` if set to - ``'{"key1": "value1", "key2": "value2"}'`` - - - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - """ - ... - - def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: - """Get a composition of a dictionary-like setting and its `_BASE` - counterpart. - - :param name: name of the dictionary-like setting - :type name: str - """ - ... - - def getpriority(self, name: _SettingsKeyT) -> Optional[int]: - """ - Return the current numerical priority value of a setting, or ``None`` if - the given ``name`` does not exist. - - :param name: the setting name - :type name: str - """ - ... - - def maxpriority(self) -> int: - """ - Return the numerical value of the highest priority present throughout - all settings, or the numerical value for ``default`` from - :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings - stored. - """ - ... - - def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: - ... - - def set(self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = ...) -> None: - """ - Store a key/value attribute with a given priority. - - Settings should be populated *before* configuring the Crawler object - (through the :meth:`~scrapy.crawler.Crawler.configure` method), - otherwise they won't have any effect. - - :param name: the setting name - :type name: str - - :param value: the value to associate with the setting - :type value: object - - :param priority: the priority of the setting. Should be a key of - :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: str or int - """ - ... - - def setdefault(self, name: _SettingsKeyT, default: Any = ..., priority: Union[int, str] = ...) -> Any: - ... - - def setdict(self, values: _SettingsInputT, priority: Union[int, str] = ...) -> None: - ... - - def setmodule(self, module: Union[ModuleType, str], priority: Union[int, str] = ...) -> None: - """ - Store settings from a module with a given priority. - - This is a helper function that calls - :meth:`~scrapy.settings.BaseSettings.set` for every globally declared - uppercase variable of ``module`` with the provided ``priority``. - - :param module: the module or the path of the module - :type module: types.ModuleType or str - - :param priority: the priority of the settings. Should be a key of - :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: str or int - """ - ... - - def update(self, values: _SettingsInputT, priority: Union[int, str] = ...) -> None: - """ - Store key/value pairs with a given priority. - - This is a helper function that calls - :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values`` - with the provided ``priority``. - - If ``values`` is a string, it is assumed to be JSON-encoded and parsed - into a dict with ``json.loads()`` first. If it is a - :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities - will be used and the ``priority`` parameter ignored. This allows - inserting/updating settings with different priorities with a single - command. - - :param values: the settings names and values - :type values: dict or string or :class:`~scrapy.settings.BaseSettings` - - :param priority: the priority of the settings. Should be a key of - :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: str or int - """ - ... - - def delete(self, name: _SettingsKeyT, priority: Union[int, str] = ...) -> None: - ... - - def __delitem__(self, name: _SettingsKeyT) -> None: - ... - - def copy(self) -> Self: - """ - Make a deep copy of current settings. - - This method returns a new instance of the :class:`Settings` class, - populated with the same values and their priorities. - - Modifications to the new object won't be reflected on the original - settings. - """ - ... - - def freeze(self) -> None: - """ - Disable further changes to the current settings. - - After calling this method, the present state of the settings will become - immutable. Trying to change values through the :meth:`~set` method and - its variants won't be possible and will be alerted. - """ - ... - - def frozencopy(self) -> Self: - """ - Return an immutable copy of the current settings. - - Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`. - """ - ... - - def __iter__(self) -> Iterator[_SettingsKeyT]: - ... - - def __len__(self) -> int: - ... - - def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: - """ - Make a copy of current settings and convert to a dict. - - This method returns a new dict populated with the same values - and their priorities as the current settings. - - Modifications to the returned dict won't be reflected on the original - settings. - - This method can be useful for example for printing settings - in Scrapy shell. - """ - ... - - def pop(self, name: _SettingsKeyT, default: Any = ...) -> Any: - ... - - - -class Settings(BaseSettings): - """ - This object stores Scrapy settings for the configuration of internal - components, and can be used for any further customization. - - It is a direct subclass and supports all methods of - :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation - of this class, the new object will have the global default settings - described on :ref:`topics-settings-ref` already populated. - """ - def __init__(self, values: _SettingsInputT = ..., priority: Union[int, str] = ...) -> None: - ... - - - -def iter_default_settings() -> Iterable[Tuple[str, Any]]: - """Return the default settings as an iterator of (name, value) tuples""" - ... - -def overridden_settings(settings: Mapping[_SettingsKeyT, Any]) -> Iterable[Tuple[str, Any]]: - """Return an iterable of the settings that have been overridden""" - ... - diff --git a/typings/scrapy/settings/default_settings.pyi b/typings/scrapy/settings/default_settings.pyi deleted file mode 100644 index 3f7997f7..00000000 --- a/typings/scrapy/settings/default_settings.pyi +++ /dev/null @@ -1,184 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import sys - -""" -This module contains the default values for all settings used by Scrapy. - -For more information about these settings you can read the settings -documentation in docs/topics/settings.rst - -Scrapy developers, if you add a setting here remember to: - -* add it in alphabetical order -* group similar settings without leaving blank lines -* add its documentation to the available settings documentation - (docs/topics/settings.rst) - -""" -ADDONS = ... -AJAXCRAWL_ENABLED = ... -ASYNCIO_EVENT_LOOP = ... -AUTOTHROTTLE_ENABLED = ... -AUTOTHROTTLE_DEBUG = ... -AUTOTHROTTLE_MAX_DELAY = ... -AUTOTHROTTLE_START_DELAY = ... -AUTOTHROTTLE_TARGET_CONCURRENCY = ... -BOT_NAME = ... -CLOSESPIDER_TIMEOUT = ... -CLOSESPIDER_PAGECOUNT = ... -CLOSESPIDER_ITEMCOUNT = ... -CLOSESPIDER_ERRORCOUNT = ... -COMMANDS_MODULE = ... -COMPRESSION_ENABLED = ... -CONCURRENT_ITEMS = ... -CONCURRENT_REQUESTS = ... -CONCURRENT_REQUESTS_PER_DOMAIN = ... -CONCURRENT_REQUESTS_PER_IP = ... -COOKIES_ENABLED = ... -COOKIES_DEBUG = ... -DEFAULT_ITEM_CLASS = ... -DEFAULT_REQUEST_HEADERS = ... -DEPTH_LIMIT = ... -DEPTH_STATS_VERBOSE = ... -DEPTH_PRIORITY = ... -DNSCACHE_ENABLED = ... -DNSCACHE_SIZE = ... -DNS_RESOLVER = ... -DNS_TIMEOUT = ... -DOWNLOAD_DELAY = ... -DOWNLOAD_HANDLERS = ... -DOWNLOAD_HANDLERS_BASE = ... -DOWNLOAD_TIMEOUT = ... -DOWNLOAD_MAXSIZE = ... -DOWNLOAD_WARNSIZE = ... -DOWNLOAD_FAIL_ON_DATALOSS = ... -DOWNLOADER = ... -DOWNLOADER_HTTPCLIENTFACTORY = ... -DOWNLOADER_CLIENTCONTEXTFACTORY = ... -DOWNLOADER_CLIENT_TLS_CIPHERS = ... -DOWNLOADER_CLIENT_TLS_METHOD = ... -DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = ... -DOWNLOADER_MIDDLEWARES = ... -DOWNLOADER_MIDDLEWARES_BASE = ... -DOWNLOADER_STATS = ... -DUPEFILTER_CLASS = ... -EDITOR = ... -if sys.platform == "win32": - ... -EXTENSIONS = ... -EXTENSIONS_BASE = ... -FEED_TEMPDIR = ... -FEEDS = ... -FEED_URI_PARAMS = ... -FEED_STORE_EMPTY = ... -FEED_EXPORT_ENCODING = ... -FEED_EXPORT_FIELDS = ... -FEED_STORAGES = ... -FEED_STORAGES_BASE = ... -FEED_EXPORT_BATCH_ITEM_COUNT = ... -FEED_EXPORTERS = ... -FEED_EXPORTERS_BASE = ... -FEED_EXPORT_INDENT = ... -FEED_STORAGE_FTP_ACTIVE = ... -FEED_STORAGE_GCS_ACL = ... -FEED_STORAGE_S3_ACL = ... -FILES_STORE_S3_ACL = ... -FILES_STORE_GCS_ACL = ... -FTP_USER = ... -FTP_PASSWORD = ... -FTP_PASSIVE_MODE = ... -GCS_PROJECT_ID = ... -HTTPCACHE_ENABLED = ... -HTTPCACHE_DIR = ... -HTTPCACHE_IGNORE_MISSING = ... -HTTPCACHE_STORAGE = ... -HTTPCACHE_EXPIRATION_SECS = ... -HTTPCACHE_ALWAYS_STORE = ... -HTTPCACHE_IGNORE_HTTP_CODES = ... -HTTPCACHE_IGNORE_SCHEMES = ... -HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = ... -HTTPCACHE_DBM_MODULE = ... -HTTPCACHE_POLICY = ... -HTTPCACHE_GZIP = ... -HTTPPROXY_ENABLED = ... -HTTPPROXY_AUTH_ENCODING = ... -IMAGES_STORE_S3_ACL = ... -IMAGES_STORE_GCS_ACL = ... -ITEM_PROCESSOR = ... -ITEM_PIPELINES = ... -ITEM_PIPELINES_BASE = ... -LOG_ENABLED = ... -LOG_ENCODING = ... -LOG_FORMATTER = ... -LOG_FORMAT = ... -LOG_DATEFORMAT = ... -LOG_STDOUT = ... -LOG_LEVEL = ... -LOG_FILE = ... -LOG_FILE_APPEND = ... -LOG_SHORT_NAMES = ... -SCHEDULER_DEBUG = ... -LOGSTATS_INTERVAL = ... -MAIL_HOST = ... -MAIL_PORT = ... -MAIL_FROM = ... -MAIL_PASS = ... -MAIL_USER = ... -MEMDEBUG_ENABLED = ... -MEMDEBUG_NOTIFY = ... -MEMUSAGE_CHECK_INTERVAL_SECONDS = ... -MEMUSAGE_ENABLED = ... -MEMUSAGE_LIMIT_MB = ... -MEMUSAGE_NOTIFY_MAIL = ... -MEMUSAGE_WARNING_MB = ... -METAREFRESH_ENABLED = ... -METAREFRESH_IGNORE_TAGS = ... -METAREFRESH_MAXDELAY = ... -NEWSPIDER_MODULE = ... -PERIODIC_LOG_DELTA = ... -PERIODIC_LOG_STATS = ... -PERIODIC_LOG_TIMING_ENABLED = ... -RANDOMIZE_DOWNLOAD_DELAY = ... -REACTOR_THREADPOOL_MAXSIZE = ... -REDIRECT_ENABLED = ... -REDIRECT_MAX_TIMES = ... -REDIRECT_PRIORITY_ADJUST = ... -REFERER_ENABLED = ... -REFERRER_POLICY = ... -REQUEST_FINGERPRINTER_CLASS = ... -REQUEST_FINGERPRINTER_IMPLEMENTATION = ... -RETRY_ENABLED = ... -RETRY_TIMES = ... -RETRY_HTTP_CODES = ... -RETRY_PRIORITY_ADJUST = ... -RETRY_EXCEPTIONS = ... -ROBOTSTXT_OBEY = ... -ROBOTSTXT_PARSER = ... -ROBOTSTXT_USER_AGENT = ... -SCHEDULER = ... -SCHEDULER_DISK_QUEUE = ... -SCHEDULER_MEMORY_QUEUE = ... -SCHEDULER_PRIORITY_QUEUE = ... -SCRAPER_SLOT_MAX_ACTIVE_SIZE = ... -SPIDER_LOADER_CLASS = ... -SPIDER_LOADER_WARN_ONLY = ... -SPIDER_MIDDLEWARES = ... -SPIDER_MIDDLEWARES_BASE = ... -SPIDER_MODULES = ... -STATS_CLASS = ... -STATS_DUMP = ... -STATSMAILER_RCPTS = ... -TEMPLATES_DIR = ... -URLLENGTH_LIMIT = ... -USER_AGENT = ... -TELNETCONSOLE_ENABLED = ... -TELNETCONSOLE_PORT = ... -TELNETCONSOLE_HOST = ... -TELNETCONSOLE_USERNAME = ... -TELNETCONSOLE_PASSWORD = ... -TWISTED_REACTOR = ... -SPIDER_CONTRACTS = ... -SPIDER_CONTRACTS_BASE = ... diff --git a/typings/scrapy/shell.pyi b/typings/scrapy/shell.pyi deleted file mode 100644 index e6641b70..00000000 --- a/typings/scrapy/shell.pyi +++ /dev/null @@ -1,35 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -"""Scrapy Shell - -See documentation in docs/topics/shell.rst - -""" -class Shell: - relevant_classes = ... - def __init__(self, crawler, update_vars=..., code=...) -> None: - ... - - def start(self, url=..., request=..., response=..., spider=..., redirect=...): # -> None: - ... - - def fetch(self, request_or_url, spider=..., redirect=..., **kwargs): # -> None: - ... - - def populate_vars(self, response=..., request=..., spider=...): # -> None: - ... - - def print_help(self): # -> None: - ... - - def get_help(self): # -> str: - ... - - - -def inspect_response(response, spider): # -> None: - """Open a shell to inspect the given response""" - ... - diff --git a/typings/scrapy/signalmanager.pyi b/typings/scrapy/signalmanager.pyi deleted file mode 100644 index 62c0c85d..00000000 --- a/typings/scrapy/signalmanager.pyi +++ /dev/null @@ -1,68 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, List, Tuple -from twisted.internet.defer import Deferred - -class SignalManager: - def __init__(self, sender: Any = ...) -> None: - ... - - def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: - """ - Connect a receiver function to a signal. - - The signal can be any object, although Scrapy comes with some - predefined signals that are documented in the :ref:`topics-signals` - section. - - :param receiver: the function to be connected - :type receiver: collections.abc.Callable - - :param signal: the signal to connect to - :type signal: object - """ - ... - - def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: - """ - Disconnect a receiver function from a signal. This has the - opposite effect of the :meth:`connect` method, and the arguments - are the same. - """ - ... - - def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: - """ - Send a signal, catch exceptions and log them. - - The keyword arguments are passed to the signal handlers (connected - through the :meth:`connect` method). - """ - ... - - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: - """ - Like :meth:`send_catch_log` but supports returning - :class:`~twisted.internet.defer.Deferred` objects from signal handlers. - - Returns a Deferred that gets fired once all signal handlers - deferreds were fired. Send a signal, catch exceptions and log them. - - The keyword arguments are passed to the signal handlers (connected - through the :meth:`connect` method). - """ - ... - - def disconnect_all(self, signal: Any, **kwargs: Any) -> None: - """ - Disconnect all receivers from the given signal. - - :param signal: the signal to disconnect from - :type signal: object - """ - ... - - - diff --git a/typings/scrapy/signals.pyi b/typings/scrapy/signals.pyi deleted file mode 100644 index 434d998e..00000000 --- a/typings/scrapy/signals.pyi +++ /dev/null @@ -1,34 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Scrapy signals - -These signals are documented in docs/topics/signals.rst. Please don't add new -signals here without documenting them there. -""" -engine_started = ... -engine_stopped = ... -spider_opened = ... -spider_idle = ... -spider_closed = ... -spider_error = ... -request_scheduled = ... -request_dropped = ... -request_reached_downloader = ... -request_left_downloader = ... -response_received = ... -response_downloaded = ... -headers_received = ... -bytes_received = ... -item_scraped = ... -item_dropped = ... -item_error = ... -feed_slot_closed = ... -feed_exporter_closed = ... -stats_spider_opened = ... -stats_spider_closing = ... -stats_spider_closed = ... -item_passed = ... -request_received = ... diff --git a/typings/scrapy/spiderloader.pyi b/typings/scrapy/spiderloader.pyi deleted file mode 100644 index 8e5f27fc..00000000 --- a/typings/scrapy/spiderloader.pyi +++ /dev/null @@ -1,47 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import List, TYPE_CHECKING, Type -from zope.interface import implementer -from scrapy import Request, Spider -from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings -from typing_extensions import Self - -if TYPE_CHECKING: - ... -@implementer(ISpiderLoader) -class SpiderLoader: - """ - SpiderLoader is a class which locates and loads spiders - in a Scrapy project. - """ - def __init__(self, settings: BaseSettings) -> None: - ... - - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - ... - - def load(self, spider_name: str) -> Type[Spider]: - """ - Return the Spider class for the given spider name. If the spider - name is not found, raise a KeyError. - """ - ... - - def find_by_request(self, request: Request) -> List[str]: - """ - Return the list of spider names that can handle the given request. - """ - ... - - def list(self) -> List[str]: - """ - Return a list with the names of all spiders available in the project. - """ - ... - - - diff --git a/typings/scrapy/spidermiddlewares/__init__.pyi b/typings/scrapy/spidermiddlewares/__init__.pyi deleted file mode 100644 index 006bc274..00000000 --- a/typings/scrapy/spidermiddlewares/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - diff --git a/typings/scrapy/spiders/__init__.pyi b/typings/scrapy/spiders/__init__.pyi deleted file mode 100644 index 9c285ab9..00000000 --- a/typings/scrapy/spiders/__init__.pyi +++ /dev/null @@ -1,75 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import logging -from __future__ import annotations -from typing import Any, Iterable, List, Optional, TYPE_CHECKING, Union, cast -from twisted.internet.defer import Deferred -from scrapy import signals -from scrapy.http import Request, Response -from scrapy.utils.trackref import object_ref -from scrapy.utils.url import url_is_from_spider -from typing_extensions import Self -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings -from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider -from scrapy.spiders.sitemap import SitemapSpider - -""" -Base class for Scrapy spiders - -See documentation in docs/topics/spiders.rst -""" -if TYPE_CHECKING: - ... -class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. - """ - name: str - custom_settings: Optional[dict] = ... - def __init__(self, name: Optional[str] = ..., **kwargs: Any) -> None: - ... - - @property - def logger(self) -> logging.LoggerAdapter: - ... - - def log(self, message: Any, level: int = ..., **kw: Any) -> None: - """Log the given message at the given log level - - This helper wraps a log call to the logger within the spider, but you - can use it directly (e.g. Spider.logger.info('msg')) or use any other - Python logger too. - """ - ... - - @classmethod - def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: - ... - - def start_requests(self) -> Iterable[Request]: - ... - - def parse(self, response: Response, **kwargs: Any) -> Any: - ... - - @classmethod - def update_settings(cls, settings: BaseSettings) -> None: - ... - - @classmethod - def handles_request(cls, request: Request) -> bool: - ... - - @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: - ... - - def __repr__(self) -> str: - ... - - - diff --git a/typings/scrapy/spiders/crawl.pyi b/typings/scrapy/spiders/crawl.pyi deleted file mode 100644 index cfbcd688..00000000 --- a/typings/scrapy/spiders/crawl.pyi +++ /dev/null @@ -1,38 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Sequence -from scrapy.http import Response -from scrapy.spiders import Spider - -""" -This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical web sites that requires crawling pages. - -See documentation in docs/topics/spiders.rst -""" -_default_link_extractor = ... -class Rule: - def __init__(self, link_extractor=..., callback=..., cb_kwargs=..., follow=..., process_links=..., process_request=..., errback=...) -> None: - ... - - - -class CrawlSpider(Spider): - rules: Sequence[Rule] = ... - def __init__(self, *a, **kw) -> None: - ... - - def parse_start_url(self, response, **kwargs): # -> list[Any]: - ... - - def process_results(self, response: Response, results: list): # -> list[Any]: - ... - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): # -> Self: - ... - - - diff --git a/typings/scrapy/spiders/feed.pyi b/typings/scrapy/spiders/feed.pyi deleted file mode 100644 index 132fc3b8..00000000 --- a/typings/scrapy/spiders/feed.pyi +++ /dev/null @@ -1,89 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.spiders import Spider - -""" -This module implements the XMLFeedSpider which is the recommended spider to use -for scraping from an XML feed. - -See documentation in docs/topics/spiders.rst -""" -class XMLFeedSpider(Spider): - """ - This class intends to be the base class for spiders that scrape - from XML feeds. - - You can choose whether to parse the file using the 'iternodes' iterator, an - 'xml' selector, or an 'html' selector. In most cases, it's convenient to - use iternodes, since it's a faster and cleaner. - """ - iterator = ... - itertag = ... - namespaces = ... - def process_results(self, response, results): - """This overridable method is called for each result (item or request) - returned by the spider, and it's intended to perform any last time - processing required before returning the results to the framework core, - for example setting the item GUIDs. It receives a list of results and - the response which originated that results. It must return a list of - results (items or requests). - """ - ... - - def adapt_response(self, response): - """You can override this function in order to make any changes you want - to into the feed before parsing it. This function must return a - response. - """ - ... - - def parse_node(self, response, selector): - """This method must be overridden with your custom spider functionality""" - ... - - def parse_nodes(self, response, nodes): # -> Generator[Any, Any, None]: - """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an Selector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either an item, a request, or a list - containing any of them. - """ - ... - - - -class CSVFeedSpider(Spider): - """Spider for parsing CSV feeds. - It receives a CSV file in a response; iterates through each of its rows, - and calls parse_row with a dict containing each field's data. - - You can set some options regarding the CSV file, such as the delimiter, quotechar - and the file's headers. - """ - delimiter = ... - quotechar = ... - headers = ... - def process_results(self, response, results): - """This method has the same purpose as the one in XMLFeedSpider""" - ... - - def adapt_response(self, response): - """This method has the same purpose as the one in XMLFeedSpider""" - ... - - def parse_row(self, response, row): - """This method must be overridden with your custom spider functionality""" - ... - - def parse_rows(self, response): # -> Generator[Any, Any, None]: - """Receives a response and a dict (representing each row) with a key for - each provided (or detected) header of the CSV file. This spider also - gives the opportunity to override adapt_response and - process_results methods for pre and post-processing purposes. - """ - ... - - - diff --git a/typings/scrapy/spiders/sitemap.pyi b/typings/scrapy/spiders/sitemap.pyi deleted file mode 100644 index b6f73541..00000000 --- a/typings/scrapy/spiders/sitemap.pyi +++ /dev/null @@ -1,33 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.spiders import Spider - -logger = ... -class SitemapSpider(Spider): - sitemap_urls = ... - sitemap_rules = ... - sitemap_follow = ... - sitemap_alternate_links = ... - def __init__(self, *a, **kw) -> None: - ... - - def start_requests(self): # -> Generator[Request, Any, None]: - ... - - def sitemap_filter(self, entries): # -> Generator[Any, Any, None]: - """This method can be used to filter sitemap entries by their - attributes, for example, you can filter locs with lastmod greater - than a given date (see docs). - """ - ... - - - -def regex(x): # -> Pattern[str]: - ... - -def iterloc(it, alt=...): # -> Generator[Any, Any, None]: - ... - diff --git a/typings/scrapy/squeues.pyi b/typings/scrapy/squeues.pyi deleted file mode 100644 index 88a8257a..00000000 --- a/typings/scrapy/squeues.pyi +++ /dev/null @@ -1,17 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -Scheduler queues -""" -_PickleFifoSerializationDiskQueue = ... -_PickleLifoSerializationDiskQueue = ... -_MarshalFifoSerializationDiskQueue = ... -_MarshalLifoSerializationDiskQueue = ... -PickleFifoDiskQueue = ... -PickleLifoDiskQueue = ... -MarshalFifoDiskQueue = ... -MarshalLifoDiskQueue = ... -FifoMemoryQueue = ... -LifoMemoryQueue = ... diff --git a/typings/scrapy/statscollectors.pyi b/typings/scrapy/statscollectors.pyi deleted file mode 100644 index d3b8b1b4..00000000 --- a/typings/scrapy/statscollectors.pyi +++ /dev/null @@ -1,78 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Dict, Optional, TYPE_CHECKING -from scrapy import Spider -from scrapy.crawler import Crawler - -""" -Scrapy extension for collecting scraping stats -""" -if TYPE_CHECKING: - ... -logger = ... -StatsT = Dict[str, Any] -class StatsCollector: - def __init__(self, crawler: Crawler) -> None: - ... - - def get_value(self, key: str, default: Any = ..., spider: Optional[Spider] = ...) -> Any: - ... - - def get_stats(self, spider: Optional[Spider] = ...) -> StatsT: - ... - - def set_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - def set_stats(self, stats: StatsT, spider: Optional[Spider] = ...) -> None: - ... - - def inc_value(self, key: str, count: int = ..., start: int = ..., spider: Optional[Spider] = ...) -> None: - ... - - def max_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - def min_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - def clear_stats(self, spider: Optional[Spider] = ...) -> None: - ... - - def open_spider(self, spider: Spider) -> None: - ... - - def close_spider(self, spider: Spider, reason: str) -> None: - ... - - - -class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: Crawler) -> None: - ... - - - -class DummyStatsCollector(StatsCollector): - def get_value(self, key: str, default: Any = ..., spider: Optional[Spider] = ...) -> Any: - ... - - def set_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - def set_stats(self, stats: StatsT, spider: Optional[Spider] = ...) -> None: - ... - - def inc_value(self, key: str, count: int = ..., start: int = ..., spider: Optional[Spider] = ...) -> None: - ... - - def max_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - def min_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None: - ... - - - diff --git a/typings/scrapy/utils/__init__.pyi b/typings/scrapy/utils/__init__.pyi deleted file mode 100644 index 006bc274..00000000 --- a/typings/scrapy/utils/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - diff --git a/typings/scrapy/utils/asyncgen.pyi b/typings/scrapy/utils/asyncgen.pyi deleted file mode 100644 index 87f3005e..00000000 --- a/typings/scrapy/utils/asyncgen.pyi +++ /dev/null @@ -1,13 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import AsyncGenerator, AsyncIterable, Iterable, Union - -async def collect_asyncgen(result: AsyncIterable) -> list: - ... - -async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: - """Wraps an iterable (sync or async) into an async generator.""" - ... - diff --git a/typings/scrapy/utils/conf.pyi b/typings/scrapy/utils/conf.pyi deleted file mode 100644 index a1921767..00000000 --- a/typings/scrapy/utils/conf.pyi +++ /dev/null @@ -1,50 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import os -from configparser import ConfigParser -from typing import Any, Callable, Dict, List, MutableMapping, Optional, Union -from scrapy.settings import BaseSettings - -def build_component_list(compdict: MutableMapping[Any, Any], custom: Any = ..., convert: Callable[[Any], Any] = ...) -> List[Any]: - """Compose a component list from a { class: order } dictionary.""" - ... - -def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: - """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a - dict - """ - ... - -def closest_scrapy_cfg(path: Union[str, os.PathLike] = ..., prevpath: Optional[Union[str, os.PathLike]] = ...) -> str: - """Return the path to the closest scrapy.cfg file by traversing the current - directory and its parents - """ - ... - -def init_env(project: str = ..., set_syspath: bool = ...) -> None: - """Initialize environment to use command-line tool from inside a project - dir. This sets the Scrapy settings module and modifies the Python path to - be able to locate the project module. - """ - ... - -def get_config(use_closest: bool = ...) -> ConfigParser: - """Get Scrapy config file as a ConfigParser""" - ... - -def get_sources(use_closest: bool = ...) -> List[str]: - ... - -def feed_complete_default_values_from_settings(feed: Dict[str, Any], settings: BaseSettings) -> Dict[str, Any]: - ... - -def feed_process_params_from_cli(settings: BaseSettings, output: List[str], output_format: Optional[str] = ..., overwrite_output: Optional[List[str]] = ...) -> Dict[str, Dict[str, Any]]: - """ - Receives feed export params (from the 'crawl' or 'runspider' commands), - checks for inconsistencies in their quantities and returns a dictionary - suitable to be used as the FEEDS setting. - """ - ... - diff --git a/typings/scrapy/utils/console.pyi b/typings/scrapy/utils/console.pyi deleted file mode 100644 index 1ad7a39f..00000000 --- a/typings/scrapy/utils/console.pyi +++ /dev/null @@ -1,17 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -DEFAULT_PYTHON_SHELLS = ... -def get_shell_embed_func(shells=..., known_shells=...): # -> _Wrapped[Callable[..., Any], _Wrapped[Callable[..., Any], Any, Callable[..., Any], None], Callable[..., Any], None] | None: - """Return the first acceptable shell-embed function - from a given list of shell names. - """ - ... - -def start_python_console(namespace=..., banner=..., shells=...): # -> None: - """Start Python console bound to the given namespace. - Readline support and tab completion will be used on Unix, if available. - """ - ... - diff --git a/typings/scrapy/utils/curl.pyi b/typings/scrapy/utils/curl.pyi deleted file mode 100644 index 7de5a891..00000000 --- a/typings/scrapy/utils/curl.pyi +++ /dev/null @@ -1,31 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import argparse - -class DataAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=...): # -> None: - ... - - - -class CurlParser(argparse.ArgumentParser): - def error(self, message): - ... - - - -curl_parser = ... -safe_to_ignore_arguments = ... -def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = ...) -> dict: - """Convert a cURL command syntax to Request kwargs. - - :param str curl_command: string containing the curl command - :param bool ignore_unknown_options: If true, only a warning is emitted when - cURL options are unknown. Otherwise - raises an error. (default: True) - :return: dictionary of Request kwargs - """ - ... - diff --git a/typings/scrapy/utils/datatypes.pyi b/typings/scrapy/utils/datatypes.pyi deleted file mode 100644 index 895b697b..00000000 --- a/typings/scrapy/utils/datatypes.pyi +++ /dev/null @@ -1,136 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import collections -import weakref -from typing import Any, AnyStr, Optional, OrderedDict, Sequence, TypeVar - -""" -This module contains data types used by Scrapy which are not included in the -Python Standard Library. - -This module must not depend on any module outside the Standard Library. -""" -_KT = TypeVar("_KT") -_VT = TypeVar("_VT") -class CaselessDict(dict): - __slots__ = ... - def __new__(cls, *args, **kwargs): # -> Self: - ... - - def __init__(self, seq=...) -> None: - ... - - def __getitem__(self, key): - ... - - def __setitem__(self, key, value): # -> None: - ... - - def __delitem__(self, key): # -> None: - ... - - def __contains__(self, key): # -> bool: - ... - - has_key = ... - def __copy__(self): # -> Self: - ... - - copy = ... - def normkey(self, key): - """Method to normalize dictionary key access""" - ... - - def normvalue(self, value): - """Method to normalize values prior to be set""" - ... - - def get(self, key, def_val=...): - ... - - def setdefault(self, key, def_val=...): - ... - - def update(self, seq): # -> None: - ... - - @classmethod - def fromkeys(cls, keys, value=...): # -> Self: - ... - - def pop(self, key, *args): - ... - - - -class CaseInsensitiveDict(collections.UserDict): - """A dict-like structure that accepts strings or bytes - as keys and allows case-insensitive lookups. - """ - def __init__(self, *args, **kwargs) -> None: - ... - - def __getitem__(self, key: AnyStr) -> Any: - ... - - def __setitem__(self, key: AnyStr, value: Any) -> None: - ... - - def __delitem__(self, key: AnyStr) -> None: - ... - - def __contains__(self, key: AnyStr) -> bool: - ... - - def __repr__(self) -> str: - ... - - - -class LocalCache(OrderedDict[_KT, _VT]): - """Dictionary with a finite number of keys. - - Older items expires first. - """ - def __init__(self, limit: Optional[int] = ...) -> None: - ... - - def __setitem__(self, key: _KT, value: _VT) -> None: - ... - - - -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): - """ - A weakref.WeakKeyDictionary implementation that uses LocalCache as its - underlying data structure, making it ordered and capable of being size-limited. - - Useful for memoization, while avoiding keeping received - arguments in memory only because of the cached references. - - Note: like LocalCache and unlike weakref.WeakKeyDictionary, - it cannot be instantiated with an initial dictionary. - """ - def __init__(self, limit: Optional[int] = ...) -> None: - ... - - def __setitem__(self, key: _KT, value: _VT) -> None: - ... - - def __getitem__(self, key: _KT) -> Optional[_VT]: - ... - - - -class SequenceExclude: - """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence) -> None: - ... - - def __contains__(self, item: Any) -> bool: - ... - - - diff --git a/typings/scrapy/utils/decorators.pyi b/typings/scrapy/utils/decorators.pyi deleted file mode 100644 index d3cba29c..00000000 --- a/typings/scrapy/utils/decorators.pyi +++ /dev/null @@ -1,23 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Callable -from twisted.internet.defer import Deferred - -def deprecated(use_instead: Any = ...) -> Callable: - """This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.""" - ... - -def defers(func: Callable) -> Callable[..., Deferred]: - """Decorator to make sure a function always returns a deferred""" - ... - -def inthread(func: Callable) -> Callable[..., Deferred]: - """Decorator to call a function in a thread and return a deferred with the - result - """ - ... - diff --git a/typings/scrapy/utils/defer.pyi b/typings/scrapy/utils/defer.pyi deleted file mode 100644 index 2d6bf403..00000000 --- a/typings/scrapy/utils/defer.pyi +++ /dev/null @@ -1,206 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from asyncio import Future -from types import CoroutineType -from typing import Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, Coroutine, Generator, Iterable, Iterator, TypeVar, Union, overload -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure - -""" -Helper functions for dealing with Twisted deferreds -""" -def defer_fail(_failure: Failure) -> Deferred: - """Same as twisted.internet.defer.fail but delay calling errback until - next reactor loop - - It delays by 100ms so reactor has a chance to go through readers and writers - before attending pending delayed calls, so do not set delay to zero. - """ - ... - -def defer_succeed(result: Any) -> Deferred: - """Same as twisted.internet.defer.succeed but delay calling callback until - next reactor loop - - It delays by 100ms so reactor has a chance to go through readers and writers - before attending pending delayed calls, so do not set delay to zero. - """ - ... - -def defer_result(result: Any) -> Deferred: - ... - -def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: - """Same as twisted.internet.defer.maybeDeferred, but delay calling - callback/errback to next reactor loop - """ - ... - -def parallel(iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any) -> Deferred: - """Execute a callable over the objects in the given iterable, in parallel, - using no more than ``count`` concurrent calls. - - Taken from: https://jcalderone.livejournal.com/24285.html - """ - ... - -class _AsyncCooperatorAdapter(Iterator): - """A class that wraps an async iterable into a normal iterator suitable - for using in Cooperator.coiterate(). As it's only needed for parallel_async(), - it calls the callable directly in the callback, instead of providing a more - generic interface. - - On the outside, this class behaves as an iterator that yields Deferreds. - Each Deferred is fired with the result of the callable which was called on - the next result from aiterator. It raises StopIteration when aiterator is - exhausted, as expected. - - Cooperator calls __next__() multiple times and waits on the Deferreds - returned from it. As async generators (since Python 3.8) don't support - awaiting on __anext__() several times in parallel, we need to serialize - this. It's done by storing the Deferreds returned from __next__() and - firing the oldest one when a result from __anext__() is available. - - The workflow: - 1. When __next__() is called for the first time, it creates a Deferred, stores it - in self.waiting_deferreds and returns it. It also makes a Deferred that will wait - for self.aiterator.__anext__() and puts it into self.anext_deferred. - 2. If __next__() is called again before self.anext_deferred fires, more Deferreds - are added to self.waiting_deferreds. - 3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both - clear self.anext_deferred. - 3.1. _callback() calls the callable passing the result value that it takes, pops a - Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it - chains those Deferreds so that the waiting Deferred will fire when the result - Deferred does, otherwise it fires it directly. This causes one awaiting task to - receive a result. If self.waiting_deferreds is still not empty, new __anext__() is - called and self.anext_deferred is populated. - 3.2. _errback() checks the exception class. If it's StopAsyncIteration it means - self.aiterator is exhausted and so it sets self.finished and fires all - self.waiting_deferreds. Other exceptions are propagated. - 4. If __next__() is called after __anext__() was handled, then if self.finished is - True, it raises StopIteration, otherwise it acts like in step 2, but if - self.anext_deferred is now empty is also populates it with a new __anext__(). - - Note that CooperativeTask ignores the value returned from the Deferred that it waits - for, so we fire them with None when needed. - - It may be possible to write an async iterator-aware replacement for - Cooperator/CooperativeTask and use it instead of this adapter to achieve the same - goal. - """ - def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args: Any, **callable_kwargs: Any) -> None: - ... - - def __next__(self) -> Deferred: - ... - - - -def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args: Any, **named: Any) -> Deferred: - """Like parallel but for async iterators""" - ... - -def process_chain(callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred: - """Return a Deferred built by chaining the given callbacks""" - ... - -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred: - """Return a Deferred built by chaining the given callbacks and errbacks""" - ... - -def process_parallel(callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred: - """Return a Deferred with the output of all successful calls to the given - callbacks - """ - ... - -def iter_errback(iterable: Iterable, errback: Callable, *a: Any, **kw: Any) -> Generator: - """Wraps an iterable calling an errback if an error is caught while - iterating it. - """ - ... - -async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any) -> AsyncGenerator: - """Wraps an async iterable calling an errback if an error is caught while - iterating it. Similar to scrapy.utils.defer.iter_errback() - """ - ... - -_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) -_T = TypeVar("_T") -@overload -def deferred_from_coro(o: _CT) -> Deferred: - ... - -@overload -def deferred_from_coro(o: _T) -> _T: - ... - -def deferred_from_coro(o: _T) -> Union[Deferred, _T]: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - ... - -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: - """Converts a coroutine function into a function that returns a Deferred. - - The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. - This is useful for callback chains, as callback functions are called with the previous callback result. - """ - ... - -def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: - """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" - ... - -def deferred_to_future(d: Deferred) -> Future: - """ - .. versionadded:: 2.6.0 - - Return an :class:`asyncio.Future` object that wraps *d*. - - When :ref:`using the asyncio reactor `, you cannot await - on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy - callables defined as coroutines `, you can only await on - ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects - allows you to wait on them:: - - class MySpider(Spider): - ... - async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await deferred_to_future(deferred) - """ - ... - -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: - """ - .. versionadded:: 2.6.0 - - Return *d* as an object that can be awaited from a :ref:`Scrapy callable - defined as a coroutine `. - - What you can await in Scrapy callables defined as coroutines depends on the - value of :setting:`TWISTED_REACTOR`: - - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - - When :ref:`using the asyncio reactor `, you can only - await on :class:`asyncio.Future` objects. - - If you want to write code that uses ``Deferred`` objects but works with any - reactor, use this function on all ``Deferred`` objects:: - - class MySpider(Spider): - ... - async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) - """ - ... - diff --git a/typings/scrapy/utils/deprecate.pyi b/typings/scrapy/utils/deprecate.pyi deleted file mode 100644 index da38d02a..00000000 --- a/typings/scrapy/utils/deprecate.pyi +++ /dev/null @@ -1,82 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Dict, List, Optional, Tuple, Type, overload - -"""Some helpers for deprecation messages""" -def attribute(obj: Any, oldattr: str, newattr: str, version: str = ...) -> None: - ... - -def create_deprecated_class(name: str, new_class: type, clsdict: Optional[Dict[str, Any]] = ..., warn_category: Type[Warning] = ..., warn_once: bool = ..., old_class_path: Optional[str] = ..., new_class_path: Optional[str] = ..., subclass_warn_message: str = ..., instance_warn_message: str = ...) -> type: - """ - Return a "deprecated" class that causes its subclasses to issue a warning. - Subclasses of ``new_class`` are considered subclasses of this class. - It also warns when the deprecated class is instantiated, but do not when - its subclasses are instantiated. - - It can be used to rename a base class in a library. For example, if we - have - - class OldName(SomeClass): - # ... - - and we want to rename it to NewName, we can do the following:: - - class NewName(SomeClass): - # ... - - OldName = create_deprecated_class('OldName', NewName) - - Then, if user class inherits from OldName, warning is issued. Also, if - some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)`` - checks they'll still return True if sub is a subclass of NewName instead of - OldName. - """ - class DeprecatedClass(new_class.__class__): - ... - - - -DEPRECATION_RULES: List[Tuple[str, str]] = ... -@overload -def update_classpath(path: str) -> str: - ... - -@overload -def update_classpath(path: Any) -> Any: - ... - -def update_classpath(path: Any) -> Any: - """Update a deprecated path from an object with its new location""" - ... - -def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool: - """ - Return True if a method named ``method_name`` of a ``base_class`` - is overridden in a ``subclass``. - - >>> class Base: - ... def foo(self): - ... pass - >>> class Sub1(Base): - ... pass - >>> class Sub2(Base): - ... def foo(self): - ... pass - >>> class Sub3(Sub1): - ... def foo(self): - ... pass - >>> class Sub4(Sub2): - ... pass - >>> method_is_overridden(Sub1, Base, 'foo') - False - >>> method_is_overridden(Sub2, Base, 'foo') - True - >>> method_is_overridden(Sub3, Base, 'foo') - True - >>> method_is_overridden(Sub4, Base, 'foo') - True - """ - ... - diff --git a/typings/scrapy/utils/gz.pyi b/typings/scrapy/utils/gz.pyi deleted file mode 100644 index 1ba6bfcc..00000000 --- a/typings/scrapy/utils/gz.pyi +++ /dev/null @@ -1,16 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.http import Response - -def gunzip(data: bytes) -> bytes: - """Gunzip the given data and return as much data as possible. - - This is resilient to CRC checksum errors. - """ - ... - -def gzip_magic_number(response: Response) -> bool: - ... - diff --git a/typings/scrapy/utils/httpobj.pyi b/typings/scrapy/utils/httpobj.pyi deleted file mode 100644 index f7d2ac43..00000000 --- a/typings/scrapy/utils/httpobj.pyi +++ /dev/null @@ -1,17 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Union -from urllib.parse import ParseResult -from weakref import WeakKeyDictionary -from scrapy.http import Request, Response - -"""Helper functions for scrapy.http objects (Request, Response)""" -_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ... -def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: - """Return urlparse.urlparse caching the result, where the argument can be a - Request or Response object - """ - ... - diff --git a/typings/scrapy/utils/iterators.pyi b/typings/scrapy/utils/iterators.pyi deleted file mode 100644 index 300eab76..00000000 --- a/typings/scrapy/utils/iterators.pyi +++ /dev/null @@ -1,51 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING, Union -from scrapy.http import Response -from scrapy.selector import Selector - -if TYPE_CHECKING: - ... -logger = ... -def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Generator[Selector, Any, None]: - """Return a iterator of Selector's over all nodes of a XML document, - given the name of the node to iterate. Useful for parsing XML feeds. - - obj can be: - - a Response object - - a unicode string - - a string encoded as utf-8 - """ - ... - -def xmliter_lxml(obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = ..., prefix: str = ...) -> Generator[Selector, Any, None]: - ... - -class _StreamReader: - def __init__(self, obj: Union[Response, str, bytes]) -> None: - ... - - def read(self, n: int = ...) -> bytes: - ... - - - -def csviter(obj: Union[Response, str, bytes], delimiter: Optional[str] = ..., headers: Optional[List[str]] = ..., encoding: Optional[str] = ..., quotechar: Optional[str] = ...) -> Generator[Dict[str, str], Any, None]: - """Returns an iterator of dictionaries from the given csv object - - obj can be: - - a Response object - - a unicode string - - a string encoded as utf-8 - - delimiter is the character used to separate fields on the given obj. - - headers is an iterable that when provided offers the keys - for the returned dictionaries, if not the first row is used. - - quotechar is the character used to enclosure fields on the given obj. - """ - ... - diff --git a/typings/scrapy/utils/job.pyi b/typings/scrapy/utils/job.pyi deleted file mode 100644 index 08ed3919..00000000 --- a/typings/scrapy/utils/job.pyi +++ /dev/null @@ -1,10 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Optional -from scrapy.settings import BaseSettings - -def job_dir(settings: BaseSettings) -> Optional[str]: - ... - diff --git a/typings/scrapy/utils/log.pyi b/typings/scrapy/utils/log.pyi deleted file mode 100644 index 7198255c..00000000 --- a/typings/scrapy/utils/log.pyi +++ /dev/null @@ -1,112 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import logging -from types import TracebackType -from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union -from twisted.python.failure import Failure -from scrapy.settings import Settings -from scrapy.crawler import Crawler - -if TYPE_CHECKING: - ... -logger = ... -def failure_to_exc_info(failure: Failure) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: - """Extract exc_info from Failure instances""" - ... - -class TopLevelFormatter(logging.Filter): - """Keep only top level loggers's name (direct children from root) from - records. - - This filter will replace Scrapy loggers' names with 'scrapy'. This mimics - the old Scrapy log behaviour and helps shortening long names. - - Since it can't be set for just one logger (it won't propagate for its - children), it's going to be set in the root handler, with a parametrized - ``loggers`` list where it should act. - """ - def __init__(self, loggers: Optional[List[str]] = ...) -> None: - ... - - def filter(self, record: logging.LogRecord) -> bool: - ... - - - -DEFAULT_LOGGING = ... -def configure_logging(settings: Union[Settings, dict, None] = ..., install_root_handler: bool = ...) -> None: - """ - Initialize logging defaults for Scrapy. - - :param settings: settings used to create and configure a handler for the - root logger (default: None). - :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None`` - - :param install_root_handler: whether to install root logging handler - (default: True) - :type install_root_handler: bool - - This function does: - - - Route warnings and twisted logging through Python standard logging - - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively - - Route stdout to log if LOG_STDOUT setting is True - - When ``install_root_handler`` is True (default), this function also - creates a handler for the root logger according to given settings - (see :ref:`topics-logging-settings`). You can override default options - using ``settings`` argument. When ``settings`` is empty or None, defaults - are used. - """ - ... - -_scrapy_root_handler: Optional[logging.Handler] = ... -def install_scrapy_root_handler(settings: Settings) -> None: - ... - -def get_scrapy_root_handler() -> Optional[logging.Handler]: - ... - -def log_scrapy_info(settings: Settings) -> None: - ... - -def log_reactor_info() -> None: - ... - -class StreamLogger: - """Fake file-like stream object that redirects writes to a logger instance - - Taken from: - https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ - """ - def __init__(self, logger: logging.Logger, log_level: int = ...) -> None: - ... - - def write(self, buf: str) -> None: - ... - - def flush(self) -> None: - ... - - - -class LogCounterHandler(logging.Handler): - """Record log levels count into a crawler stats""" - def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any) -> None: - ... - - def emit(self, record: logging.LogRecord) -> None: - ... - - - -def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: - """ - Helper that takes the dictionary output from the methods in LogFormatter - and adapts it into a tuple of positional arguments for logger.log calls, - handling backward compatibility as well. - """ - ... - diff --git a/typings/scrapy/utils/misc.pyi b/typings/scrapy/utils/misc.pyi deleted file mode 100644 index 59d32c71..00000000 --- a/typings/scrapy/utils/misc.pyi +++ /dev/null @@ -1,112 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import ast -from contextlib import contextmanager -from types import ModuleType -from typing import Any, Callable, Generator, IO, Iterable, List, Optional, Pattern, TYPE_CHECKING, Union -from scrapy import Spider - -"""Helper functions which don't fit anywhere else""" -if TYPE_CHECKING: - ... -_ITERABLE_SINGLE_VALUES = ... -def arg_to_iter(arg: Any) -> Iterable[Any]: - """Convert an argument to an iterable. The argument can be a None, single - value, or an iterable. - - Exception: if arg is a dict, [arg] will be returned - """ - ... - -def load_object(path: Union[str, Callable]) -> Any: - """Load an object given its absolute object path, and return it. - - The object can be the import path of a class, function, variable or an - instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'. - - If ``path`` is not a string, but is a callable object, such as a class or - a function, then return it as is. - """ - ... - -def walk_modules(path: str) -> List[ModuleType]: - """Loads a module and all its submodules from the given module path and - returns them. If *any* module throws an exception while importing, that - exception is thrown back. - - For example: walk_modules('scrapy.utils') - """ - ... - -def extract_regex(regex: Union[str, Pattern], text: str, encoding: str = ...) -> List[str]: - """Extract a list of unicode strings from the given text/encoding using the following policies: - - * if the regex contains a named group called "extract" that will be returned - * if the regex contains multiple numbered groups, all those will be returned (flattened) - * if the regex doesn't contain any group the entire regex matching is returned - """ - ... - -def md5sum(file: IO) -> str: - """Calculate the md5 checksum of a file-like object without reading its - whole content in memory. - - >>> from io import BytesIO - >>> md5sum(BytesIO(b'file content to hash')) - '784406af91dd5a54fbb9c84c2236595a' - """ - ... - -def rel_has_nofollow(rel: Optional[str]) -> bool: - """Return True if link rel attribute has nofollow type""" - ... - -def create_instance(objcls, settings, crawler, *args, **kwargs): - """Construct a class instance using its ``from_crawler`` or - ``from_settings`` constructors, if available. - - At least one of ``settings`` and ``crawler`` needs to be different from - ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` constructor will be - tried. - - ``*args`` and ``**kwargs`` are forwarded to the constructors. - - Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. - - .. versionchanged:: 2.2 - Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an - extension has not been implemented correctly). - """ - ... - -@contextmanager -def set_environ(**kwargs: str) -> Generator[None, Any, None]: - """Temporarily set environment variables inside the context manager and - fully restore previous environment afterwards - """ - ... - -def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: - """Similar to ``ast.walk``, but walks only function body and skips nested - functions defined within the node. - """ - ... - -_generator_callbacks_cache = ... -def is_generator_with_return_value(callable: Callable) -> bool: - """ - Returns True if a callable is a generator function which includes a - 'return' statement with a value different than None, False otherwise - """ - ... - -def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: - """ - Logs a warning if a callable is a generator function and includes - a 'return' statement with a value different than None - """ - ... - diff --git a/typings/scrapy/utils/ossignal.pyi b/typings/scrapy/utils/ossignal.pyi deleted file mode 100644 index ca6e3ded..00000000 --- a/typings/scrapy/utils/ossignal.pyi +++ /dev/null @@ -1,18 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import signal -from types import FrameType -from typing import Any, Callable, Dict, Optional, Union - -SignalHandlerT = Union[Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None] -signal_names: Dict[int, str] = ... -def install_shutdown_handlers(function: SignalHandlerT, override_sigint: bool = ...) -> None: - """Install the given function as a signal handler for all common shutdown - signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the - SIGINT handler won't be install if there is already a handler in place - (e.g. Pdb) - """ - ... - diff --git a/typings/scrapy/utils/project.pyi b/typings/scrapy/utils/project.pyi deleted file mode 100644 index 6e9b1e07..00000000 --- a/typings/scrapy/utils/project.pyi +++ /dev/null @@ -1,25 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from scrapy.settings import Settings - -ENVVAR = ... -DATADIR_CFG_SECTION = ... -def inside_project() -> bool: - ... - -def project_data_dir(project: str = ...) -> str: - """Return the current project data dir, creating it if it doesn't exist""" - ... - -def data_path(path: str, createdir: bool = ...) -> str: - """ - Return the given path joined with the .scrapy data directory. - If given an absolute path, return it unmodified. - """ - ... - -def get_project_settings() -> Settings: - ... - diff --git a/typings/scrapy/utils/python.pyi b/typings/scrapy/utils/python.pyi deleted file mode 100644 index c6a7105f..00000000 --- a/typings/scrapy/utils/python.pyi +++ /dev/null @@ -1,201 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import sys -from typing import Any, AsyncIterable, AsyncIterator, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, Pattern, Tuple, Union, overload - -""" -This module contains essential stuff that should've come with Python itself ;) -""" -def flatten(x: Iterable) -> list: - """flatten(sequence) -> list - - Returns a single, flat list which contains all elements retrieved - from the sequence and all recursively contained sub-sequences - (iterables). - - Examples: - >>> [1, 2, [3,4], (5,6)] - [1, 2, [3, 4], (5, 6)] - >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10] - >>> flatten(["foo", "bar"]) - ['foo', 'bar'] - >>> flatten(["foo", ["baz", 42], "bar"]) - ['foo', 'baz', 42, 'bar'] - """ - ... - -def iflatten(x: Iterable) -> Iterable: - """iflatten(sequence) -> iterator - - Similar to ``.flatten()``, but returns iterator instead""" - ... - -def is_listlike(x: Any) -> bool: - """ - >>> is_listlike("foo") - False - >>> is_listlike(5) - False - >>> is_listlike(b"foo") - False - >>> is_listlike([b"foo"]) - True - >>> is_listlike((b"foo",)) - True - >>> is_listlike({}) - True - >>> is_listlike(set()) - True - >>> is_listlike((x for x in range(3))) - True - >>> is_listlike(range(5)) - True - """ - ... - -def unique(list_: Iterable, key: Callable[[Any], Any] = ...) -> list: - """efficient function to uniquify a list preserving item order""" - ... - -def to_unicode(text: Union[str, bytes], encoding: Optional[str] = ..., errors: str = ...) -> str: - """Return the unicode representation of a bytes object ``text``. If - ``text`` is already an unicode object, return it as-is.""" - ... - -def to_bytes(text: Union[str, bytes], encoding: Optional[str] = ..., errors: str = ...) -> bytes: - """Return the binary representation of ``text``. If ``text`` - is already a bytes object, return it as-is.""" - ... - -def re_rsearch(pattern: Union[str, Pattern], text: str, chunk_size: int = ...) -> Optional[Tuple[int, int]]: - """ - This function does a reverse search in a text using a regular expression - given in the attribute 'pattern'. - Since the re module does not provide this functionality, we have to find for - the expression into chunks of text extracted from the end (for the sake of efficiency). - At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for - the pattern. If the pattern is not found, another chunk is extracted, and another - search is performed. - This process continues until a match is found, or until the whole file is read. - In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing - the start position of the match, and the ending (regarding the entire text). - """ - ... - -def memoizemethod_noargs(method: Callable) -> Callable: - """Decorator to cache the result of a method (without arguments) using a - weak reference to its object - """ - ... - -_BINARYCHARS = ... -def binary_is_text(data: bytes) -> bool: - """Returns ``True`` if the given ``data`` argument (a ``bytes`` object) - does not contain unprintable control characters. - """ - ... - -def get_func_args(func: Callable, stripself: bool = ...) -> List[str]: - """Return the argument name list of a callable object""" - ... - -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: - """Returns (args, kwargs) tuple for a function - >>> import re - >>> get_spec(re.match) - (['pattern', 'string'], {'flags': 0}) - - >>> class Test: - ... def __call__(self, val): - ... pass - ... def method(self, val, flags=0): - ... pass - - >>> get_spec(Test) - (['self', 'val'], {}) - - >>> get_spec(Test.method) - (['self', 'val'], {'flags': 0}) - - >>> get_spec(Test().method) - (['self', 'val'], {'flags': 0}) - """ - ... - -def equal_attributes(obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]) -> bool: - """Compare two objects attributes""" - ... - -@overload -def without_none_values(iterable: Mapping) -> dict: - ... - -@overload -def without_none_values(iterable: Iterable) -> Iterable: - ... - -def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: - """Return a copy of ``iterable`` with all ``None`` entries removed. - - If ``iterable`` is a mapping, return a dictionary where all pairs that have - value ``None`` have been removed. - """ - ... - -def global_object_name(obj: Any) -> str: - """ - Return full name of a global object. - - >>> from scrapy import Request - >>> global_object_name(Request) - 'scrapy.http.request.Request' - """ - ... - -if hasattr(sys, "pypy_version_info"): - def garbage_collect() -> None: - ... - -else: - def garbage_collect() -> None: - ... - -class MutableChain(Iterable): - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ - def __init__(self, *args: Iterable) -> None: - ... - - def extend(self, *iterables: Iterable) -> None: - ... - - def __iter__(self) -> Iterator: - ... - - def __next__(self) -> Any: - ... - - - -class MutableAsyncChain(AsyncIterable): - """ - Similar to MutableChain but for async iterables - """ - def __init__(self, *args: Union[Iterable, AsyncIterable]) -> None: - ... - - def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: - ... - - def __aiter__(self) -> AsyncIterator: - ... - - async def __anext__(self) -> Any: - ... - - - diff --git a/typings/scrapy/utils/reactor.pyi b/typings/scrapy/utils/reactor.pyi deleted file mode 100644 index 8ecd8247..00000000 --- a/typings/scrapy/utils/reactor.pyi +++ /dev/null @@ -1,61 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy -from typing import Any, Callable, Optional - -def listen_tcp(portrange, host, factory): # -> None: - """Like reactor.listenTCP but tries different ports in a range.""" - ... - -class CallLaterOnce: - """Schedule a function to be called in the next reactor loop, but only if - it hasn't been already scheduled since the last time it ran. - """ - def __init__(self, func: Callable, *a: Any, **kw: Any) -> None: - ... - - def schedule(self, delay: float = ...) -> None: - ... - - def cancel(self) -> None: - ... - - def __call__(self) -> Any: - ... - - - -def set_asyncio_event_loop_policy() -> None: - """The policy functions from asyncio often behave unexpectedly, - so we restrict their use to the absolutely essential case. - This should only be used to install the reactor. - """ - ... - -def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: - ... - -def install_reactor(reactor_path: str, event_loop_path: Optional[str] = ...) -> None: - """Installs the :mod:`~twisted.internet.reactor` with the specified - import path. Also installs the asyncio event loop with the specified import - path if the asyncio reactor is enabled""" - ... - -def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: - """Sets and returns the event loop with specified import path.""" - ... - -def verify_installed_reactor(reactor_path: str) -> None: - """Raises :exc:`Exception` if the installed - :mod:`~twisted.internet.reactor` does not match the specified import - path.""" - ... - -def verify_installed_asyncio_event_loop(loop_path: str) -> None: - ... - -def is_asyncio_reactor_installed() -> bool: - ... - diff --git a/typings/scrapy/utils/request.pyi b/typings/scrapy/utils/request.pyi deleted file mode 100644 index 1cdd56fb..00000000 --- a/typings/scrapy/utils/request.pyi +++ /dev/null @@ -1,150 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Dict, Iterable, Optional, Protocol, TYPE_CHECKING, Tuple, Union -from weakref import WeakKeyDictionary -from scrapy import Request, Spider -from scrapy.crawler import Crawler - -""" -This module provides some useful functions for working with -scrapy.http.Request objects -""" -if TYPE_CHECKING: - ... -_deprecated_fingerprint_cache: WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]] -_deprecated_fingerprint_cache = ... -def request_fingerprint(request: Request, include_headers: Optional[Iterable[Union[bytes, str]]] = ..., keep_fragments: bool = ...) -> str: - """ - Return the request fingerprint as an hexadecimal string. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - ... - -_fingerprint_cache: WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]] -_fingerprint_cache = ... -def fingerprint(request: Request, *, include_headers: Optional[Iterable[Union[bytes, str]]] = ..., keep_fragments: bool = ...) -> bytes: - """ - Return the request fingerprint. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - ... - -class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... - - - -class RequestFingerprinter: - """Default fingerprinter. - - It takes into account a canonical version - (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url - ` and the values of :attr:`request.method - ` and :attr:`request.body - `. It then generates an `SHA1 - `_ hash. - - .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. - """ - @classmethod - def from_crawler(cls, crawler): # -> Self: - ... - - def __init__(self, crawler: Optional[Crawler] = ...) -> None: - ... - - def fingerprint(self, request: Request) -> bytes: - ... - - - -def request_authenticate(request: Request, username: str, password: str) -> None: - """Authenticate the given request (in place) using the HTTP basic access - authentication mechanism (RFC 2617) and the given username and password - """ - ... - -def request_httprepr(request: Request) -> bytes: - """Return the raw HTTP representation (as bytes) of the given request. - This is provided only for reference since it's not the actual stream of - bytes that will be send when performing the request (that's controlled - by Twisted). - """ - ... - -def referer_str(request: Request) -> Optional[str]: - """Return Referer HTTP header suitable for logging.""" - ... - -def request_from_dict(d: dict, *, spider: Optional[Spider] = ...) -> Request: - """Create a :class:`~scrapy.Request` object from a dict. - - If a spider is given, it will try to resolve the callbacks looking at the - spider for methods with the same name. - """ - ... - -def request_to_curl(request: Request) -> str: - """ - Converts a :class:`~scrapy.Request` object to a curl command. - - :param :class:`~scrapy.Request`: Request object to be converted - :return: string containing the curl command - """ - ... - diff --git a/typings/scrapy/utils/response.pyi b/typings/scrapy/utils/response.pyi deleted file mode 100644 index b9de172c..00000000 --- a/typings/scrapy/utils/response.pyi +++ /dev/null @@ -1,44 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import scrapy -from typing import Any, Callable, Iterable, Tuple, Union -from weakref import WeakKeyDictionary -from twisted.web import http -from w3lib import html -from scrapy.http.response import Response -from scrapy.utils.decorators import deprecated - -""" -This module provides some useful functions for working with -scrapy.http.Response objects -""" -_baseurl_cache: WeakKeyDictionary[Response, str] = ... -def get_base_url(response: scrapy.http.response.text.TextResponse) -> str: - """Return the base url of the given response, joined with the response url""" - ... - -_metaref_cache: WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]] = ... -def get_meta_refresh(response: scrapy.http.response.text.TextResponse, ignore_tags: Iterable[str] = ...) -> Union[Tuple[None, None], Tuple[float, str]]: - """Parse the http-equiv refresh parameter from the given response""" - ... - -def response_status_message(status: Union[bytes, float, int, str]) -> str: - """Return status code plus status text descriptive message""" - ... - -@deprecated -def response_httprepr(response: Response) -> bytes: - """Return raw HTTP representation (as bytes) of the given response. This - is provided only for reference, since it's not the exact stream of bytes - that was received (that's not exposed by Twisted). - """ - ... - -def open_in_browser(response: Union[scrapy.http.response.html.HtmlResponse, scrapy.http.response.text.TextResponse,], _openfunc: Callable[[str], Any] = ...) -> Any: - """Open the given response in a local web browser, populating the - tag for external links to work - """ - ... - diff --git a/typings/scrapy/utils/serialize.pyi b/typings/scrapy/utils/serialize.pyi deleted file mode 100644 index 7baa404e..00000000 --- a/typings/scrapy/utils/serialize.pyi +++ /dev/null @@ -1,19 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import json -from typing import Any - -class ScrapyJSONEncoder(json.JSONEncoder): - DATE_FORMAT = ... - TIME_FORMAT = ... - def default(self, o: Any) -> Any: - ... - - - -class ScrapyJSONDecoder(json.JSONDecoder): - ... - - diff --git a/typings/scrapy/utils/signal.pyi b/typings/scrapy/utils/signal.pyi deleted file mode 100644 index 6a92c416..00000000 --- a/typings/scrapy/utils/signal.pyi +++ /dev/null @@ -1,28 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any as TypingAny, List, Tuple -from twisted.internet.defer import Deferred - -"""Helper functions for working with signals""" -logger = ... -def send_catch_log(signal: TypingAny = ..., sender: TypingAny = ..., *arguments: TypingAny, **named: TypingAny) -> List[Tuple[TypingAny, TypingAny]]: - """Like pydispatcher.robust.sendRobust but it also logs errors and returns - Failures instead of exceptions. - """ - ... - -def send_catch_log_deferred(signal: TypingAny = ..., sender: TypingAny = ..., *arguments: TypingAny, **named: TypingAny) -> Deferred: - """Like send_catch_log but supports returning deferreds on signal handlers. - Returns a deferred that gets fired once all signal handlers deferreds were - fired. - """ - ... - -def disconnect_all(signal: TypingAny = ..., sender: TypingAny = ...) -> None: - """Disconnect all signal handlers. Useful for cleaning up after running - tests - """ - ... - diff --git a/typings/scrapy/utils/sitemap.pyi b/typings/scrapy/utils/sitemap.pyi deleted file mode 100644 index 23c1d6a7..00000000 --- a/typings/scrapy/utils/sitemap.pyi +++ /dev/null @@ -1,29 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Dict, Generator, Iterator, Optional - -""" -Module for processing Sitemaps. - -Note: The main purpose of this module is to provide support for the -SitemapSpider, its API is subject to change without notice. -""" -class Sitemap: - """Class to parse Sitemap (type=urlset) and Sitemap Index - (type=sitemapindex) files""" - def __init__(self, xmltext: str) -> None: - ... - - def __iter__(self) -> Iterator[Dict[str, Any]]: - ... - - - -def sitemap_urls_from_robots(robots_text: str, base_url: Optional[str] = ...) -> Generator[str, Any, None]: - """Return an iterator over all sitemap urls contained in the given - robots.txt file - """ - ... - diff --git a/typings/scrapy/utils/spider.pyi b/typings/scrapy/utils/spider.pyi deleted file mode 100644 index f8fb56c0..00000000 --- a/typings/scrapy/utils/spider.pyi +++ /dev/null @@ -1,65 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from types import CoroutineType, ModuleType -from typing import Any, AsyncGenerator, Generator, Iterable, Literal, Optional, TYPE_CHECKING, Type, TypeVar, Union, overload -from twisted.internet.defer import Deferred -from scrapy import Request -from scrapy.spiders import Spider -from scrapy.spiderloader import SpiderLoader - -if TYPE_CHECKING: - ... -logger = ... -_T = TypeVar("_T") -@overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: - ... - -@overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... - -@overload -def iterate_spider_output(result: _T) -> Iterable: - ... - -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: - ... - -def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: - """Return an iterator over all spider classes defined in the given module - that can be instantiated (i.e. which have name) - """ - ... - -@overload -def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ...) -> Type[Spider]: - ... - -@overload -def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]: - ... - -@overload -def spidercls_for_request(spider_loader: SpiderLoader, request: Request, *, log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]: - ... - -def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Optional[Type[Spider]] = ..., log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]: - """Return a spider class that handles the given Request. - - This will look for the spiders that can handle the given request (using - the spider loader) and return a Spider class if (and only if) there is - only one Spider able to handle the Request. - - If multiple spiders (or no spider) are found, it will return the - default_spidercls passed. It can optionally log if multiple or no spiders - are found. - """ - ... - -class DefaultSpider(Spider): - name = ... - - diff --git a/typings/scrapy/utils/ssl.pyi b/typings/scrapy/utils/ssl.pyi deleted file mode 100644 index c66dc976..00000000 --- a/typings/scrapy/utils/ssl.pyi +++ /dev/null @@ -1,19 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, Optional -from OpenSSL.crypto import X509Name - -def ffi_buf_to_string(buf: Any) -> str: - ... - -def x509name_to_string(x509name: X509Name) -> str: - ... - -def get_temp_key_info(ssl_object: Any) -> Optional[str]: - ... - -def get_openssl_version() -> str: - ... - diff --git a/typings/scrapy/utils/trackref.pyi b/typings/scrapy/utils/trackref.pyi deleted file mode 100644 index 6cde7725..00000000 --- a/typings/scrapy/utils/trackref.pyi +++ /dev/null @@ -1,46 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Any, DefaultDict, Iterable, TYPE_CHECKING -from weakref import WeakKeyDictionary -from typing_extensions import Self - -"""This module provides some functions and classes to record and report -references to live object instances. - -If you want live objects for a particular class to be tracked, you only have to -subclass from object_ref (instead of object). - -About performance: This library has a minimal performance impact when enabled, -and no performance penalty at all when disabled (as object_ref becomes just an -alias to object in that case). -""" -if TYPE_CHECKING: - ... -NoneType = ... -live_refs: DefaultDict[type, WeakKeyDictionary] = ... -class object_ref: - """Inherit from this class to a keep a record of live instances""" - __slots__ = ... - def __new__(cls, *args: Any, **kwargs: Any) -> Self: - ... - - - -def format_live_refs(ignore: Any = ...) -> str: - """Return a tabular representation of tracked objects""" - ... - -def print_live_refs(*a: Any, **kw: Any) -> None: - """Print tracked objects""" - ... - -def get_oldest(class_name: str) -> Any: - """Get the oldest object for a specific class name""" - ... - -def iter_all(class_name: str) -> Iterable[Any]: - """Iterate over all objects of the same class by its class name""" - ... - diff --git a/typings/scrapy/utils/url.pyi b/typings/scrapy/utils/url.pyi deleted file mode 100644 index 27862434..00000000 --- a/typings/scrapy/utils/url.pyi +++ /dev/null @@ -1,83 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import Iterable, Optional, TYPE_CHECKING, Type, Union -from urllib.parse import ParseResult -from w3lib.url import * -from scrapy import Spider - -""" -This module contains general purpose URL functions not found in the standard -library. - -Some of the functions that used to be imported from this module have been moved -to the w3lib.url module. Always import those from there instead. -""" -if TYPE_CHECKING: - ... -UrlT = Union[str, bytes, ParseResult] -def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: - """Return True if the url belongs to any of the given domains""" - ... - -def url_is_from_spider(url: UrlT, spider: Type[Spider]) -> bool: - """Return True if the url belongs to the given spider""" - ... - -def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: - """Return True if the url ends with one of the extensions provided""" - ... - -def parse_url(url: UrlT, encoding: Optional[str] = ...) -> ParseResult: - """Return urlparsed url from the given argument (which could be an already - parsed url) - """ - ... - -def escape_ajax(url: str) -> str: - """ - Return the crawlable url according to: - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started - - >>> escape_ajax("www.example.com/ajax.html#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value") - 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html#!") - 'www.example.com/ajax.html?_escaped_fragment_=' - - URLs that are not "AJAX crawlable" (according to Google) returned as-is: - - >>> escape_ajax("www.example.com/ajax.html#key=value") - 'www.example.com/ajax.html#key=value' - >>> escape_ajax("www.example.com/ajax.html#") - 'www.example.com/ajax.html#' - >>> escape_ajax("www.example.com/ajax.html") - 'www.example.com/ajax.html' - """ - ... - -def add_http_if_no_scheme(url: str) -> str: - """Add http as the default scheme if it is missing from the url.""" - ... - -def guess_scheme(url: str) -> str: - """Add an URL scheme if missing: file:// for filepath-like input or - http:// otherwise.""" - ... - -def strip_url(url: str, strip_credentials: bool = ..., strip_default_port: bool = ..., origin_only: bool = ..., strip_fragment: bool = ...) -> str: - """Strip URL string from some of its components: - - - ``strip_credentials`` removes "user:password@" - - ``strip_default_port`` removes ":80" (resp. ":443", ":21") - from http:// (resp. https://, ftp://) URLs - - ``origin_only`` replaces path component with "/", also dropping - query and fragment components ; it also strips credentials - - ``strip_fragment`` drops any #fragment component - """ - ... - diff --git a/typings/scrapy/utils/versions.pyi b/typings/scrapy/utils/versions.pyi deleted file mode 100644 index cf99dbbb..00000000 --- a/typings/scrapy/utils/versions.pyi +++ /dev/null @@ -1,9 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from typing import List, Tuple - -def scrapy_components_versions() -> List[Tuple[str, str]]: - ... - diff --git a/typings/toolz/__init__.pyi b/typings/toolz/__init__.pyi deleted file mode 100644 index 1108f52b..00000000 --- a/typings/toolz/__init__.pyi +++ /dev/null @@ -1,17 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from .itertoolz import * -from .functoolz import * -from .dicttoolz import * -from .recipes import * -from functools import partial, reduce -from . import curried, sandbox -from ._version import get_versions - -sorted = ... -map = map -filter = filter -comp = ... -__version__ = get_versions()['version'] diff --git a/typings/toolz/_signatures.pyi b/typings/toolz/_signatures.pyi deleted file mode 100644 index a01daa22..00000000 --- a/typings/toolz/_signatures.pyi +++ /dev/null @@ -1,73 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -"""Internal module for better introspection of builtins. - -The main functions are ``is_builtin_valid_args``, ``is_builtin_partial_args``, -and ``has_unknown_args``. Other functions in this module support these three. - -Notably, we create a ``signatures`` registry to enable introspection of -builtin functions in any Python version. This includes builtins that -have more than one valid signature. Currently, the registry includes -builtins from ``builtins``, ``functools``, ``itertools``, and ``operator`` -modules. More can be added as requested. We don't guarantee full coverage. - -Everything in this module should be regarded as implementation details. -Users should try to not use this module directly. -""" -module_info = ... -def num_pos_args(sigspec): # -> int: - """ Return the number of positional arguments. ``f(x, y=1)`` has 1""" - ... - -def get_exclude_keywords(num_pos_only, sigspec): # -> tuple[()] | tuple[Unknown, ...]: - """ Return the names of position-only arguments if func has **kwargs""" - ... - -def signature_or_spec(func): # -> Signature | None: - ... - -def expand_sig(sig): # -> tuple[Unknown | int, Unknown, Unknown | tuple[Any, ...], Signature | None]: - """ Convert the signature spec in ``module_info`` to add to ``signatures`` - - The input signature spec is one of: - - ``lambda_func`` - - ``(num_position_args, lambda_func)`` - - ``(num_position_args, lambda_func, keyword_only_args)`` - - The output signature spec is: - ``(num_position_args, lambda_func, keyword_exclude, sigspec)`` - - where ``keyword_exclude`` includes keyword only arguments and, if variadic - keywords is present, the names of position-only argument. The latter is - included to support builtins such as ``partial(func, *args, **kwargs)``, - which allows ``func=`` to be used as a keyword even though it's the name - of a positional argument. - """ - ... - -signatures = ... -def create_signature_registry(module_info=..., signatures=...): # -> None: - ... - -def check_valid(sig, args, kwargs): # -> bool: - """ Like ``is_valid_args`` for the given signature spec""" - ... - -def check_partial(sig, args, kwargs): - """ Like ``is_partial_args`` for the given signature spec""" - ... - -def check_arity(n, sig): # -> bool | None: - ... - -def check_varargs(sig): # -> bool | None: - ... - -def check_keywords(sig): # -> bool | None: - ... - -def check_required_args(sig): # -> int | Literal[False] | None: - ... - diff --git a/typings/toolz/_version.pyi b/typings/toolz/_version.pyi deleted file mode 100644 index 28e6f13f..00000000 --- a/typings/toolz/_version.pyi +++ /dev/null @@ -1,8 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -version_json = ... -def get_versions(): # -> Any: - ... - diff --git a/typings/toolz/curried/__init__.pyi b/typings/toolz/curried/__init__.pyi deleted file mode 100644 index 321c6226..00000000 --- a/typings/toolz/curried/__init__.pyi +++ /dev/null @@ -1,77 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import toolz -from . import operator -from toolz import apply, comp, complement, compose, compose_left, concat, concatv, count, curry, diff, first, flip, frequencies, identity, interleave, isdistinct, isiterable, juxt, last, memoize, merge_sorted, peek, pipe, second, thread_first, thread_last -from .exceptions import merge, merge_with - -""" -Alternate namespace for toolz such that all functions are curried - -Currying provides implicit partial evaluation of all functions - -Example: - - Get usually requires two arguments, an index and a collection - >>> from toolz.curried import get - >>> get(0, ('a', 'b')) - 'a' - - When we use it in higher order functions we often want to pass a partially - evaluated form - >>> data = [(1, 2), (11, 22), (111, 222)] - >>> list(map(lambda seq: get(0, seq), data)) - [1, 11, 111] - - The curried version allows simple expression of partial evaluation - >>> list(map(get(0), data)) - [1, 11, 111] - -See Also: - toolz.functoolz.curry -""" -accumulate = ... -assoc = ... -assoc_in = ... -cons = ... -countby = ... -dissoc = ... -do = ... -drop = ... -excepts = ... -filter = ... -get = ... -get_in = ... -groupby = ... -interpose = ... -itemfilter = ... -itemmap = ... -iterate = ... -join = ... -keyfilter = ... -keymap = ... -map = ... -mapcat = ... -nth = ... -partial = ... -partition = ... -partition_all = ... -partitionby = ... -peekn = ... -pluck = ... -random_sample = ... -reduce = ... -reduceby = ... -remove = ... -sliding_window = ... -sorted = ... -tail = ... -take = ... -take_nth = ... -topk = ... -unique = ... -update_in = ... -valfilter = ... -valmap = ... diff --git a/typings/toolz/curried/exceptions.pyi b/typings/toolz/curried/exceptions.pyi deleted file mode 100644 index 85684966..00000000 --- a/typings/toolz/curried/exceptions.pyi +++ /dev/null @@ -1,15 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -import toolz - -__all__ = ['merge_with', 'merge'] -@toolz.curry -def merge_with(func, d, *dicts, **kwargs): # -> dict[Unknown, Unknown]: - ... - -@toolz.curry -def merge(d, *dicts, **kwargs): # -> dict[Unknown, Unknown]: - ... - diff --git a/typings/toolz/curried/operator.pyi b/typings/toolz/curried/operator.pyi deleted file mode 100644 index aed5435d..00000000 --- a/typings/toolz/curried/operator.pyi +++ /dev/null @@ -1,5 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -IGNORE = ... diff --git a/typings/toolz/dicttoolz.pyi b/typings/toolz/dicttoolz.pyi deleted file mode 100644 index f16f92ca..00000000 --- a/typings/toolz/dicttoolz.pyi +++ /dev/null @@ -1,235 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -__all__ = ('merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'assoc_in', 'update_in', 'get_in') -def merge(*dicts, **kwargs): # -> dict[Unknown, Unknown]: - """ Merge a collection of dictionaries - - >>> merge({1: 'one'}, {2: 'two'}) - {1: 'one', 2: 'two'} - - Later dictionaries have precedence - - >>> merge({1: 2, 3: 4}, {3: 3, 4: 4}) - {1: 2, 3: 3, 4: 4} - - See Also: - merge_with - """ - ... - -def merge_with(func, *dicts, **kwargs): # -> dict[Unknown, Unknown]: - """ Merge dictionaries and apply function to combined values - - A key may occur in more than one dict, and all values mapped from the key - will be passed to the function as a list, such as func([val1, val2, ...]). - - >>> merge_with(sum, {1: 1, 2: 2}, {1: 10, 2: 20}) - {1: 11, 2: 22} - - >>> merge_with(first, {1: 1, 2: 2}, {2: 20, 3: 30}) # doctest: +SKIP - {1: 1, 2: 2, 3: 30} - - See Also: - merge - """ - ... - -def valmap(func, d, factory=...): # -> dict[Unknown, Unknown]: - """ Apply function to values of dictionary - - >>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]} - >>> valmap(sum, bills) # doctest: +SKIP - {'Alice': 65, 'Bob': 45} - - See Also: - keymap - itemmap - """ - ... - -def keymap(func, d, factory=...): # -> dict[Unknown, Unknown]: - """ Apply function to keys of dictionary - - >>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]} - >>> keymap(str.lower, bills) # doctest: +SKIP - {'alice': [20, 15, 30], 'bob': [10, 35]} - - See Also: - valmap - itemmap - """ - ... - -def itemmap(func, d, factory=...): # -> dict[Unknown, Unknown]: - """ Apply function to items of dictionary - - >>> accountids = {"Alice": 10, "Bob": 20} - >>> itemmap(reversed, accountids) # doctest: +SKIP - {10: "Alice", 20: "Bob"} - - See Also: - keymap - valmap - """ - ... - -def valfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]: - """ Filter items in dictionary by value - - >>> iseven = lambda x: x % 2 == 0 - >>> d = {1: 2, 2: 3, 3: 4, 4: 5} - >>> valfilter(iseven, d) - {1: 2, 3: 4} - - See Also: - keyfilter - itemfilter - valmap - """ - ... - -def keyfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]: - """ Filter items in dictionary by key - - >>> iseven = lambda x: x % 2 == 0 - >>> d = {1: 2, 2: 3, 3: 4, 4: 5} - >>> keyfilter(iseven, d) - {2: 3, 4: 5} - - See Also: - valfilter - itemfilter - keymap - """ - ... - -def itemfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]: - """ Filter items in dictionary by item - - >>> def isvalid(item): - ... k, v = item - ... return k % 2 == 0 and v < 4 - - >>> d = {1: 2, 2: 3, 3: 4, 4: 5} - >>> itemfilter(isvalid, d) - {2: 3} - - See Also: - keyfilter - valfilter - itemmap - """ - ... - -def assoc(d, key, value, factory=...): # -> dict[Unknown, Unknown]: - """ Return a new dict with new key value pair - - New dict has d[key] set to value. Does not modify the initial dictionary. - - >>> assoc({'x': 1}, 'x', 2) - {'x': 2} - >>> assoc({'x': 1}, 'y', 3) # doctest: +SKIP - {'x': 1, 'y': 3} - """ - ... - -def dissoc(d, *keys, **kwargs): # -> dict[Unknown, Unknown]: - """ Return a new dict with the given key(s) removed. - - New dict has d[key] deleted for each supplied key. - Does not modify the initial dictionary. - - >>> dissoc({'x': 1, 'y': 2}, 'y') - {'x': 1} - >>> dissoc({'x': 1, 'y': 2}, 'y', 'x') - {} - >>> dissoc({'x': 1}, 'y') # Ignores missing keys - {'x': 1} - """ - ... - -def assoc_in(d, keys, value, factory=...): # -> dict[Unknown, Unknown]: - """ Return a new dict with new, potentially nested, key value pair - - >>> purchase = {'name': 'Alice', - ... 'order': {'items': ['Apple', 'Orange'], - ... 'costs': [0.50, 1.25]}, - ... 'credit card': '5555-1234-1234-1234'} - >>> assoc_in(purchase, ['order', 'costs'], [0.25, 1.00]) # doctest: +SKIP - {'credit card': '5555-1234-1234-1234', - 'name': 'Alice', - 'order': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}} - """ - ... - -def update_in(d, keys, func, default=..., factory=...): # -> dict[Unknown, Unknown]: - """ Update value in a (potentially) nested dictionary - - inputs: - d - dictionary on which to operate - keys - list or tuple giving the location of the value to be changed in d - func - function to operate on that value - - If keys == [k0,..,kX] and d[k0]..[kX] == v, update_in returns a copy of the - original dictionary with v replaced by func(v), but does not mutate the - original dictionary. - - If k0 is not a key in d, update_in creates nested dictionaries to the depth - specified by the keys, with the innermost value set to func(default). - - >>> inc = lambda x: x + 1 - >>> update_in({'a': 0}, ['a'], inc) - {'a': 1} - - >>> transaction = {'name': 'Alice', - ... 'purchase': {'items': ['Apple', 'Orange'], - ... 'costs': [0.50, 1.25]}, - ... 'credit card': '5555-1234-1234-1234'} - >>> update_in(transaction, ['purchase', 'costs'], sum) # doctest: +SKIP - {'credit card': '5555-1234-1234-1234', - 'name': 'Alice', - 'purchase': {'costs': 1.75, 'items': ['Apple', 'Orange']}} - - >>> # updating a value when k0 is not in d - >>> update_in({}, [1, 2, 3], str, default="bar") - {1: {2: {3: 'bar'}}} - >>> update_in({1: 'foo'}, [2, 3, 4], inc, 0) - {1: 'foo', 2: {3: {4: 1}}} - """ - ... - -def get_in(keys, coll, default=..., no_default=...): # -> Sequence[Unknown] | None: - """ Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys. - - If coll[i0][i1]...[iX] cannot be found, returns ``default``, unless - ``no_default`` is specified, then it raises KeyError or IndexError. - - ``get_in`` is a generalization of ``operator.getitem`` for nested data - structures such as dictionaries and lists. - - >>> transaction = {'name': 'Alice', - ... 'purchase': {'items': ['Apple', 'Orange'], - ... 'costs': [0.50, 1.25]}, - ... 'credit card': '5555-1234-1234-1234'} - >>> get_in(['purchase', 'items', 0], transaction) - 'Apple' - >>> get_in(['name'], transaction) - 'Alice' - >>> get_in(['purchase', 'total'], transaction) - >>> get_in(['purchase', 'items', 'apple'], transaction) - >>> get_in(['purchase', 'items', 10], transaction) - >>> get_in(['purchase', 'total'], transaction, 0) - 0 - >>> get_in(['y'], {}, no_default=True) - Traceback (most recent call last): - ... - KeyError: 'y' - - See Also: - itertoolz.get - operator.getitem - """ - ... - diff --git a/typings/toolz/functoolz.pyi b/typings/toolz/functoolz.pyi deleted file mode 100644 index 89bdf91d..00000000 --- a/typings/toolz/functoolz.pyi +++ /dev/null @@ -1,540 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -""" -This type stub file was generated by pyright. -""" -PYPY = ... -__all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', 'compose', 'compose_left', 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', 'excepts') -PYPY = ... -def identity(x): - """ Identity function. Return x - - >>> identity(3) - 3 - """ - ... - -def apply(*func_and_args, **kwargs): - """ Applies a function and returns the results - - >>> def double(x): return 2*x - >>> def inc(x): return x + 1 - >>> apply(double, 5) - 10 - - >>> tuple(map(apply, [double, inc, double], [10, 500, 8000])) - (20, 501, 16000) - """ - ... - -def thread_first(val, *forms): - """ Thread value through a sequence of functions/forms - - >>> def double(x): return 2*x - >>> def inc(x): return x + 1 - >>> thread_first(1, inc, double) - 4 - - If the function expects more than one input you can specify those inputs - in a tuple. The value is used as the first input. - - >>> def add(x, y): return x + y - >>> def pow(x, y): return x**y - >>> thread_first(1, (add, 4), (pow, 2)) # pow(add(1, 4), 2) - 25 - - So in general - thread_first(x, f, (g, y, z)) - expands to - g(f(x), y, z) - - See Also: - thread_last - """ - ... - -def thread_last(val, *forms): - """ Thread value through a sequence of functions/forms - - >>> def double(x): return 2*x - >>> def inc(x): return x + 1 - >>> thread_last(1, inc, double) - 4 - - If the function expects more than one input you can specify those inputs - in a tuple. The value is used as the last input. - - >>> def add(x, y): return x + y - >>> def pow(x, y): return x**y - >>> thread_last(1, (add, 4), (pow, 2)) # pow(2, add(4, 1)) - 32 - - So in general - thread_last(x, f, (g, y, z)) - expands to - g(y, z, f(x)) - - >>> def iseven(x): - ... return x % 2 == 0 - >>> list(thread_last([1, 2, 3], (map, inc), (filter, iseven))) - [2, 4] - - See Also: - thread_first - """ - ... - -def instanceproperty(fget=..., fset=..., fdel=..., doc=..., classval=...): - """ Like @property, but returns ``classval`` when used as a class attribute - - >>> class MyClass(object): - ... '''The class docstring''' - ... @instanceproperty(classval=__doc__) - ... def __doc__(self): - ... return 'An object docstring' - ... @instanceproperty - ... def val(self): - ... return 42 - ... - >>> MyClass.__doc__ - 'The class docstring' - >>> MyClass.val is None - True - >>> obj = MyClass() - >>> obj.__doc__ - 'An object docstring' - >>> obj.val - 42 - """ - ... - -class InstanceProperty(property): - """ Like @property, but returns ``classval`` when used as a class attribute - - Should not be used directly. Use ``instanceproperty`` instead. - """ - def __init__(self, fget=..., fset=..., fdel=..., doc=..., classval=...) -> None: - ... - - def __get__(self, obj, type=...): - ... - - def __reduce__(self): - ... - - - -class curry: - """ Curry a callable function - - Enables partial application of arguments through calling a function with an - incomplete set of arguments. - - >>> def mul(x, y): - ... return x * y - >>> mul = curry(mul) - - >>> double = mul(2) - >>> double(10) - 20 - - Also supports keyword arguments - - >>> @curry # Can use curry as a decorator - ... def f(x, y, a=10): - ... return a * (x + y) - - >>> add = f(a=1) - >>> add(2, 3) - 5 - - See Also: - toolz.curried - namespace of curried functions - https://toolz.readthedocs.io/en/latest/curry.html - """ - def __init__(self, *args, **kwargs) -> None: - ... - - @instanceproperty - def func(self): - ... - - @instanceproperty - def __signature__(self): - ... - - @instanceproperty - def args(self): - ... - - @instanceproperty - def keywords(self): - ... - - @instanceproperty - def func_name(self): - ... - - def __str__(self) -> str: - ... - - def __repr__(self): - ... - - def __hash__(self) -> int: - ... - - def __eq__(self, other) -> bool: - ... - - def __ne__(self, other) -> bool: - ... - - def __call__(self, *args, **kwargs): - ... - - def bind(self, *args, **kwargs): - ... - - def call(self, *args, **kwargs): - ... - - def __get__(self, instance, owner): - ... - - def __reduce__(self): - ... - - - -@curry -def memoize(func, cache=..., key=...): - """ Cache a function's result for speedy future evaluation - - Considerations: - Trades memory for speed. - Only use on pure functions. - - >>> def add(x, y): return x + y - >>> add = memoize(add) - - Or use as a decorator - - >>> @memoize - ... def add(x, y): - ... return x + y - - Use the ``cache`` keyword to provide a dict-like object as an initial cache - - >>> @memoize(cache={(1, 2): 3}) - ... def add(x, y): - ... return x + y - - Note that the above works as a decorator because ``memoize`` is curried. - - It is also possible to provide a ``key(args, kwargs)`` function that - calculates keys used for the cache, which receives an ``args`` tuple and - ``kwargs`` dict as input, and must return a hashable value. However, - the default key function should be sufficient most of the time. - - >>> # Use key function that ignores extraneous keyword arguments - >>> @memoize(key=lambda args, kwargs: args) - ... def add(x, y, verbose=False): - ... if verbose: - ... print('Calculating %s + %s' % (x, y)) - ... return x + y - """ - ... - -class Compose: - """ A composition of functions - - See Also: - compose - """ - __slots__ = ... - def __init__(self, funcs) -> None: - ... - - def __call__(self, *args, **kwargs): - ... - - def __getstate__(self): - ... - - def __setstate__(self, state): - ... - - @instanceproperty(classval=__doc__) - def __doc__(self): - ... - - @property - def __name__(self): - ... - - def __repr__(self): - ... - - def __eq__(self, other) -> bool: - ... - - def __ne__(self, other) -> bool: - ... - - def __hash__(self) -> int: - ... - - def __get__(self, obj, objtype=...): - ... - - @instanceproperty - def __signature__(self): - ... - - __wrapped__ = ... - - -def compose(*funcs): - """ Compose functions to operate in series. - - Returns a function that applies other functions in sequence. - - Functions are applied from right to left so that - ``compose(f, g, h)(x, y)`` is the same as ``f(g(h(x, y)))``. - - If no arguments are provided, the identity function (f(x) = x) is returned. - - >>> inc = lambda i: i + 1 - >>> compose(str, inc)(3) - '4' - - See Also: - compose_left - pipe - """ - ... - -def compose_left(*funcs): - """ Compose functions to operate in series. - - Returns a function that applies other functions in sequence. - - Functions are applied from left to right so that - ``compose_left(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``. - - If no arguments are provided, the identity function (f(x) = x) is returned. - - >>> inc = lambda i: i + 1 - >>> compose_left(inc, str)(3) - '4' - - See Also: - compose - pipe - """ - ... - -def pipe(data, *funcs): - """ Pipe a value through a sequence of functions - - I.e. ``pipe(data, f, g, h)`` is equivalent to ``h(g(f(data)))`` - - We think of the value as progressing through a pipe of several - transformations, much like pipes in UNIX - - ``$ cat data | f | g | h`` - - >>> double = lambda i: 2 * i - >>> pipe(3, double, str) - '6' - - See Also: - compose - compose_left - thread_first - thread_last - """ - ... - -def complement(func): - """ Convert a predicate function to its logical complement. - - In other words, return a function that, for inputs that normally - yield True, yields False, and vice-versa. - - >>> def iseven(n): return n % 2 == 0 - >>> isodd = complement(iseven) - >>> iseven(2) - True - >>> isodd(2) - False - """ - ... - -class juxt: - """ Creates a function that calls several functions with the same arguments - - Takes several functions and returns a function that applies its arguments - to each of those functions then returns a tuple of the results. - - Name comes from juxtaposition: the fact of two things being seen or placed - close together with contrasting effect. - - >>> inc = lambda x: x + 1 - >>> double = lambda x: x * 2 - >>> juxt(inc, double)(10) - (11, 20) - >>> juxt([inc, double])(10) - (11, 20) - """ - __slots__ = ... - def __init__(self, *funcs) -> None: - ... - - def __call__(self, *args, **kwargs): - ... - - def __getstate__(self): - ... - - def __setstate__(self, state): - ... - - - -def do(func, x): - """ Runs ``func`` on ``x``, returns ``x`` - - Because the results of ``func`` are not returned, only the side - effects of ``func`` are relevant. - - Logging functions can be made by composing ``do`` with a storage function - like ``list.append`` or ``file.write`` - - >>> from toolz import compose - >>> from toolz.curried import do - - >>> log = [] - >>> inc = lambda x: x + 1 - >>> inc = compose(inc, do(log.append)) - >>> inc(1) - 2 - >>> inc(11) - 12 - >>> log - [1, 11] - """ - ... - -@curry -def flip(func, a, b): - """ Call the function call with the arguments flipped - - This function is curried. - - >>> def div(a, b): - ... return a // b - ... - >>> flip(div, 2, 6) - 3 - >>> div_by_two = flip(div, 2) - >>> div_by_two(4) - 2 - - This is particularly useful for built in functions and functions defined - in C extensions that accept positional only arguments. For example: - isinstance, issubclass. - - >>> data = [1, 'a', 'b', 2, 1.5, object(), 3] - >>> only_ints = list(filter(flip(isinstance, int), data)) - >>> only_ints - [1, 2, 3] - """ - ... - -def return_none(exc): - """ Returns None. - """ - ... - -class excepts: - """A wrapper around a function to catch exceptions and - dispatch to a handler. - - This is like a functional try/except block, in the same way that - ifexprs are functional if/else blocks. - - Examples - -------- - >>> excepting = excepts( - ... ValueError, - ... lambda a: [1, 2].index(a), - ... lambda _: -1, - ... ) - >>> excepting(1) - 0 - >>> excepting(3) - -1 - - Multiple exceptions and default except clause. - >>> excepting = excepts((IndexError, KeyError), lambda a: a[0]) - >>> excepting([]) - >>> excepting([1]) - 1 - >>> excepting({}) - >>> excepting({0: 1}) - 1 - """ - def __init__(self, exc, func, handler=...) -> None: - ... - - def __call__(self, *args, **kwargs): - ... - - @instanceproperty(classval=__doc__) - def __doc__(self): - ... - - @property - def __name__(self): - ... - - - -if PYPY: - _check_sigspec_orig = ... -def num_required_args(func, sigspec=...): - ... - -def has_varargs(func, sigspec=...): - ... - -def has_keywords(func, sigspec=...): - ... - -def is_valid_args(func, args, kwargs, sigspec=...): - ... - -def is_partial_args(func, args, kwargs, sigspec=...): - ... - -def is_arity(n, func, sigspec=...): - """ Does a function have only n positional arguments? - - This function relies on introspection and does not call the function. - Returns None if validity can't be determined. - - >>> def f(x): - ... return x - >>> is_arity(1, f) - True - >>> def g(x, y=1): - ... return x + y - >>> is_arity(1, g) - False - """ - ... - diff --git a/typings/toolz/itertoolz.pyi b/typings/toolz/itertoolz.pyi deleted file mode 100644 index a60dbea9..00000000 --- a/typings/toolz/itertoolz.pyi +++ /dev/null @@ -1,666 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -__all__ = ('remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'unique', 'isiterable', 'isdistinct', 'take', 'drop', 'take_nth', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', 'join', 'tail', 'diff', 'topk', 'peek', 'peekn', 'random_sample') -def remove(predicate, seq): # -> filterfalse[Unknown]: - """ Return those items of sequence for which predicate(item) is False - - >>> def iseven(x): - ... return x % 2 == 0 - >>> list(remove(iseven, [1, 2, 3, 4])) - [1, 3] - """ - ... - -def accumulate(binop, seq, initial=...): # -> Generator[Unknown | str, Any, None]: - """ Repeatedly apply binary function to a sequence, accumulating results - - >>> from operator import add, mul - >>> list(accumulate(add, [1, 2, 3, 4, 5])) - [1, 3, 6, 10, 15] - >>> list(accumulate(mul, [1, 2, 3, 4, 5])) - [1, 2, 6, 24, 120] - - Accumulate is similar to ``reduce`` and is good for making functions like - cumulative sum: - - >>> from functools import partial, reduce - >>> sum = partial(reduce, add) - >>> cumsum = partial(accumulate, add) - - Accumulate also takes an optional argument that will be used as the first - value. This is similar to reduce. - - >>> list(accumulate(add, [1, 2, 3], -1)) - [-1, 0, 2, 5] - >>> list(accumulate(add, [], 1)) - [1] - - See Also: - itertools.accumulate : In standard itertools for Python 3.2+ - """ - ... - -def groupby(key, seq): # -> dict[Unknown, Unknown]: - """ Group a collection by a key function - - >>> names = ['Alice', 'Bob', 'Charlie', 'Dan', 'Edith', 'Frank'] - >>> groupby(len, names) # doctest: +SKIP - {3: ['Bob', 'Dan'], 5: ['Alice', 'Edith', 'Frank'], 7: ['Charlie']} - - >>> iseven = lambda x: x % 2 == 0 - >>> groupby(iseven, [1, 2, 3, 4, 5, 6, 7, 8]) # doctest: +SKIP - {False: [1, 3, 5, 7], True: [2, 4, 6, 8]} - - Non-callable keys imply grouping on a member. - - >>> groupby('gender', [{'name': 'Alice', 'gender': 'F'}, - ... {'name': 'Bob', 'gender': 'M'}, - ... {'name': 'Charlie', 'gender': 'M'}]) # doctest:+SKIP - {'F': [{'gender': 'F', 'name': 'Alice'}], - 'M': [{'gender': 'M', 'name': 'Bob'}, - {'gender': 'M', 'name': 'Charlie'}]} - - Not to be confused with ``itertools.groupby`` - - See Also: - countby - """ - ... - -def merge_sorted(*seqs, **kwargs): # -> Iterator[Any] | Generator[Unknown, Any, None]: - """ Merge and sort a collection of sorted collections - - This works lazily and only keeps one value from each iterable in memory. - - >>> list(merge_sorted([1, 3, 5], [2, 4, 6])) - [1, 2, 3, 4, 5, 6] - - >>> ''.join(merge_sorted('abc', 'abc', 'abc')) - 'aaabbbccc' - - The "key" function used to sort the input may be passed as a keyword. - - >>> list(merge_sorted([2, 3], [1, 3], key=lambda x: x // 3)) - [2, 1, 3, 3] - """ - ... - -def interleave(seqs): # -> Generator[Unknown, Any, None]: - """ Interleave a sequence of sequences - - >>> list(interleave([[1, 2], [3, 4]])) - [1, 3, 2, 4] - - >>> ''.join(interleave(('ABC', 'XY'))) - 'AXBYC' - - Both the individual sequences and the sequence of sequences may be infinite - - Returns a lazy iterator - """ - ... - -def unique(seq, key=...): # -> Generator[Unknown, Any, None]: - """ Return only unique elements of a sequence - - >>> tuple(unique((1, 2, 3))) - (1, 2, 3) - >>> tuple(unique((1, 2, 1, 3))) - (1, 2, 3) - - Uniqueness can be defined by key keyword - - >>> tuple(unique(['cat', 'mouse', 'dog', 'hen'], key=len)) - ('cat', 'mouse') - """ - ... - -def isiterable(x): # -> bool: - """ Is x iterable? - - >>> isiterable([1, 2, 3]) - True - >>> isiterable('abc') - True - >>> isiterable(5) - False - """ - ... - -def isdistinct(seq): # -> bool: - """ All values in sequence are distinct - - >>> isdistinct([1, 2, 3]) - True - >>> isdistinct([1, 2, 1]) - False - - >>> isdistinct("Hello") - False - >>> isdistinct("World") - True - """ - ... - -def take(n, seq): # -> islice[Unknown]: - """ The first n elements of a sequence - - >>> list(take(2, [10, 20, 30, 40, 50])) - [10, 20] - - See Also: - drop - tail - """ - ... - -def tail(n, seq): # -> tuple[Unknown, ...]: - """ The last n elements of a sequence - - >>> tail(2, [10, 20, 30, 40, 50]) - [40, 50] - - See Also: - drop - take - """ - ... - -def drop(n, seq): # -> islice[Unknown]: - """ The sequence following the first n elements - - >>> list(drop(2, [10, 20, 30, 40, 50])) - [30, 40, 50] - - See Also: - take - tail - """ - ... - -def take_nth(n, seq): # -> islice[Unknown]: - """ Every nth item in seq - - >>> list(take_nth(2, [10, 20, 30, 40, 50])) - [10, 30, 50] - """ - ... - -def first(seq): - """ The first element in a sequence - - >>> first('ABC') - 'A' - """ - ... - -def second(seq): - """ The second element in a sequence - - >>> second('ABC') - 'B' - """ - ... - -def nth(n, seq): - """ The nth element in a sequence - - >>> nth(1, 'ABC') - 'B' - """ - ... - -def last(seq): - """ The last element in a sequence - - >>> last('ABC') - 'C' - """ - ... - -rest = ... -def get(ind, seq, default=...): # -> Any | tuple[Unknown] | tuple[()] | tuple[Unknown | str, ...] | str: - """ Get element in a sequence or dict - - Provides standard indexing - - >>> get(1, 'ABC') # Same as 'ABC'[1] - 'B' - - Pass a list to get multiple values - - >>> get([1, 2], 'ABC') # ('ABC'[1], 'ABC'[2]) - ('B', 'C') - - Works on any value that supports indexing/getitem - For example here we see that it works with dictionaries - - >>> phonebook = {'Alice': '555-1234', - ... 'Bob': '555-5678', - ... 'Charlie':'555-9999'} - >>> get('Alice', phonebook) - '555-1234' - - >>> get(['Alice', 'Bob'], phonebook) - ('555-1234', '555-5678') - - Provide a default for missing values - - >>> get(['Alice', 'Dennis'], phonebook, None) - ('555-1234', None) - - See Also: - pluck - """ - ... - -def concat(seqs): # -> chain[Unknown]: - """ Concatenate zero or more iterables, any of which may be infinite. - - An infinite sequence will prevent the rest of the arguments from - being included. - - We use chain.from_iterable rather than ``chain(*seqs)`` so that seqs - can be a generator. - - >>> list(concat([[], [1], [2, 3]])) - [1, 2, 3] - - See also: - itertools.chain.from_iterable equivalent - """ - ... - -def concatv(*seqs): # -> chain[Unknown]: - """ Variadic version of concat - - >>> list(concatv([], ["a"], ["b", "c"])) - ['a', 'b', 'c'] - - See also: - itertools.chain - """ - ... - -def mapcat(func, seqs): # -> chain[Unknown]: - """ Apply func to each sequence in seqs, concatenating results. - - >>> list(mapcat(lambda s: [c.upper() for c in s], - ... [["a", "b"], ["c", "d", "e"]])) - ['A', 'B', 'C', 'D', 'E'] - """ - ... - -def cons(el, seq): # -> chain[Unknown]: - """ Add el to beginning of (possibly infinite) sequence seq. - - >>> list(cons(1, [2, 3])) - [1, 2, 3] - """ - ... - -def interpose(el, seq): # -> chain[Unknown]: - """ Introduce element between each pair of elements in seq - - >>> list(interpose("a", [1, 2, 3])) - [1, 'a', 2, 'a', 3] - """ - ... - -def frequencies(seq): # -> dict[Unknown, int]: - """ Find number of occurrences of each value in seq - - >>> frequencies(['cat', 'cat', 'ox', 'pig', 'pig', 'cat']) #doctest: +SKIP - {'cat': 3, 'ox': 1, 'pig': 2} - - See Also: - countby - groupby - """ - ... - -def reduceby(key, binop, seq, init=...): # -> dict[Unknown, Unknown]: - """ Perform a simultaneous groupby and reduction - - The computation: - - >>> result = reduceby(key, binop, seq, init) # doctest: +SKIP - - is equivalent to the following: - - >>> def reduction(group): # doctest: +SKIP - ... return reduce(binop, group, init) # doctest: +SKIP - - >>> groups = groupby(key, seq) # doctest: +SKIP - >>> result = valmap(reduction, groups) # doctest: +SKIP - - But the former does not build the intermediate groups, allowing it to - operate in much less space. This makes it suitable for larger datasets - that do not fit comfortably in memory - - The ``init`` keyword argument is the default initialization of the - reduction. This can be either a constant value like ``0`` or a callable - like ``lambda : 0`` as might be used in ``defaultdict``. - - Simple Examples - --------------- - - >>> from operator import add, mul - >>> iseven = lambda x: x % 2 == 0 - - >>> data = [1, 2, 3, 4, 5] - - >>> reduceby(iseven, add, data) # doctest: +SKIP - {False: 9, True: 6} - - >>> reduceby(iseven, mul, data) # doctest: +SKIP - {False: 15, True: 8} - - Complex Example - --------------- - - >>> projects = [{'name': 'build roads', 'state': 'CA', 'cost': 1000000}, - ... {'name': 'fight crime', 'state': 'IL', 'cost': 100000}, - ... {'name': 'help farmers', 'state': 'IL', 'cost': 2000000}, - ... {'name': 'help farmers', 'state': 'CA', 'cost': 200000}] - - >>> reduceby('state', # doctest: +SKIP - ... lambda acc, x: acc + x['cost'], - ... projects, 0) - {'CA': 1200000, 'IL': 2100000} - - Example Using ``init`` - ---------------------- - - >>> def set_add(s, i): - ... s.add(i) - ... return s - - >>> reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2, 3], set) # doctest: +SKIP - {True: set([2, 4]), - False: set([1, 3])} - """ - ... - -def iterate(func, x): # -> Generator[Unknown, Any, None]: - """ Repeatedly apply a function func onto an original input - - Yields x, then func(x), then func(func(x)), then func(func(func(x))), etc.. - - >>> def inc(x): return x + 1 - >>> counter = iterate(inc, 0) - >>> next(counter) - 0 - >>> next(counter) - 1 - >>> next(counter) - 2 - - >>> double = lambda x: x * 2 - >>> powers_of_two = iterate(double, 1) - >>> next(powers_of_two) - 1 - >>> next(powers_of_two) - 2 - >>> next(powers_of_two) - 4 - >>> next(powers_of_two) - 8 - """ - ... - -def sliding_window(n, seq): # -> zip[tuple[Unknown]]: - """ A sequence of overlapping subsequences - - >>> list(sliding_window(2, [1, 2, 3, 4])) - [(1, 2), (2, 3), (3, 4)] - - This function creates a sliding window suitable for transformations like - sliding means / smoothing - - >>> mean = lambda seq: float(sum(seq)) / len(seq) - >>> list(map(mean, sliding_window(2, [1, 2, 3, 4]))) - [1.5, 2.5, 3.5] - """ - ... - -no_pad = ... -def partition(n, seq, pad=...): # -> zip[Unknown] | zip_longest[Unknown]: - """ Partition sequence into tuples of length n - - >>> list(partition(2, [1, 2, 3, 4])) - [(1, 2), (3, 4)] - - If the length of ``seq`` is not evenly divisible by ``n``, the final tuple - is dropped if ``pad`` is not specified, or filled to length ``n`` by pad: - - >>> list(partition(2, [1, 2, 3, 4, 5])) - [(1, 2), (3, 4)] - - >>> list(partition(2, [1, 2, 3, 4, 5], pad=None)) - [(1, 2), (3, 4), (5, None)] - - See Also: - partition_all - """ - ... - -def partition_all(n, seq): # -> Generator[Unknown, Any, None]: - """ Partition all elements of sequence into tuples of length at most n - - The final tuple may be shorter to accommodate extra elements. - - >>> list(partition_all(2, [1, 2, 3, 4])) - [(1, 2), (3, 4)] - - >>> list(partition_all(2, [1, 2, 3, 4, 5])) - [(1, 2), (3, 4), (5,)] - - See Also: - partition - """ - ... - -def count(seq): # -> int: - """ Count the number of items in seq - - Like the builtin ``len`` but works on lazy sequences. - - Not to be confused with ``itertools.count`` - - See also: - len - """ - ... - -def pluck(ind, seqs, default=...): # -> map[Any] | Generator[tuple[Unknown | str, ...], None, None] | Generator[Unknown | str, None, None]: - """ plucks an element or several elements from each item in a sequence. - - ``pluck`` maps ``itertoolz.get`` over a sequence and returns one or more - elements of each item in the sequence. - - This is equivalent to running `map(curried.get(ind), seqs)` - - ``ind`` can be either a single string/index or a list of strings/indices. - ``seqs`` should be sequence containing sequences or dicts. - - e.g. - - >>> data = [{'id': 1, 'name': 'Cheese'}, {'id': 2, 'name': 'Pies'}] - >>> list(pluck('name', data)) - ['Cheese', 'Pies'] - >>> list(pluck([0, 1], [[1, 2, 3], [4, 5, 7]])) - [(1, 2), (4, 5)] - - See Also: - get - map - """ - ... - -def getter(index): # -> ((x: Unknown) -> tuple[Unknown]) | itemgetter[Unknown] | ((x: Unknown) -> tuple[()]): - ... - -def join(leftkey, leftseq, rightkey, rightseq, left_default=..., right_default=...): # -> Generator[tuple[Unknown, Unknown] | tuple[str, Unknown] | tuple[Unknown, str], Any, None]: - """ Join two sequences on common attributes - - This is a semi-streaming operation. The LEFT sequence is fully evaluated - and placed into memory. The RIGHT sequence is evaluated lazily and so can - be arbitrarily large. - (Note: If right_default is defined, then unique keys of rightseq - will also be stored in memory.) - - >>> friends = [('Alice', 'Edith'), - ... ('Alice', 'Zhao'), - ... ('Edith', 'Alice'), - ... ('Zhao', 'Alice'), - ... ('Zhao', 'Edith')] - - >>> cities = [('Alice', 'NYC'), - ... ('Alice', 'Chicago'), - ... ('Dan', 'Syndey'), - ... ('Edith', 'Paris'), - ... ('Edith', 'Berlin'), - ... ('Zhao', 'Shanghai')] - - >>> # Vacation opportunities - >>> # In what cities do people have friends? - >>> result = join(second, friends, - ... first, cities) - >>> for ((a, b), (c, d)) in sorted(unique(result)): - ... print((a, d)) - ('Alice', 'Berlin') - ('Alice', 'Paris') - ('Alice', 'Shanghai') - ('Edith', 'Chicago') - ('Edith', 'NYC') - ('Zhao', 'Chicago') - ('Zhao', 'NYC') - ('Zhao', 'Berlin') - ('Zhao', 'Paris') - - Specify outer joins with keyword arguments ``left_default`` and/or - ``right_default``. Here is a full outer join in which unmatched elements - are paired with None. - - >>> identity = lambda x: x - >>> list(join(identity, [1, 2, 3], - ... identity, [2, 3, 4], - ... left_default=None, right_default=None)) - [(2, 2), (3, 3), (None, 4), (1, None)] - - Usually the key arguments are callables to be applied to the sequences. If - the keys are not obviously callable then it is assumed that indexing was - intended, e.g. the following is a legal change. - The join is implemented as a hash join and the keys of leftseq must be - hashable. Additionally, if right_default is defined, then keys of rightseq - must also be hashable. - - >>> # result = join(second, friends, first, cities) - >>> result = join(1, friends, 0, cities) # doctest: +SKIP - """ - ... - -def diff(*seqs, **kwargs): # -> Generator[Unknown, Any, None]: - """ Return those items that differ between sequences - - >>> list(diff([1, 2, 3], [1, 2, 10, 100])) - [(3, 10)] - - Shorter sequences may be padded with a ``default`` value: - - >>> list(diff([1, 2, 3], [1, 2, 10, 100], default=None)) - [(3, 10), (None, 100)] - - A ``key`` function may also be applied to each item to use during - comparisons: - - >>> list(diff(['apples', 'bananas'], ['Apples', 'Oranges'], key=str.lower)) - [('bananas', 'Oranges')] - """ - ... - -def topk(k, seq, key=...): # -> tuple[Unknown, ...]: - """ Find the k largest elements of a sequence - - Operates lazily in ``n*log(k)`` time - - >>> topk(2, [1, 100, 10, 1000]) - (1000, 100) - - Use a key function to change sorted order - - >>> topk(2, ['Alice', 'Bob', 'Charlie', 'Dan'], key=len) - ('Charlie', 'Alice') - - See also: - heapq.nlargest - """ - ... - -def peek(seq): # -> tuple[Unknown, chain[Unknown]]: - """ Retrieve the next element of a sequence - - Returns the first element and an iterable equivalent to the original - sequence, still having the element retrieved. - - >>> seq = [0, 1, 2, 3, 4] - >>> first, seq = peek(seq) - >>> first - 0 - >>> list(seq) - [0, 1, 2, 3, 4] - """ - ... - -def peekn(n, seq): # -> tuple[tuple[Unknown, ...], chain[Unknown]]: - """ Retrieve the next n elements of a sequence - - Returns a tuple of the first n elements and an iterable equivalent - to the original, still having the elements retrieved. - - >>> seq = [0, 1, 2, 3, 4] - >>> first_two, seq = peekn(2, seq) - >>> first_two - (0, 1) - >>> list(seq) - [0, 1, 2, 3, 4] - """ - ... - -def random_sample(prob, seq, random_state=...): # -> filter[Unknown]: - """ Return elements from a sequence with probability of prob - - Returns a lazy iterator of random items from seq. - - ``random_sample`` considers each item independently and without - replacement. See below how the first time it returned 13 items and the - next time it returned 6 items. - - >>> seq = list(range(100)) - >>> list(random_sample(0.1, seq)) # doctest: +SKIP - [6, 9, 19, 35, 45, 50, 58, 62, 68, 72, 78, 86, 95] - >>> list(random_sample(0.1, seq)) # doctest: +SKIP - [6, 44, 54, 61, 69, 94] - - Providing an integer seed for ``random_state`` will result in - deterministic sampling. Given the same seed it will return the same sample - every time. - - >>> list(random_sample(0.1, seq, random_state=2016)) - [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] - >>> list(random_sample(0.1, seq, random_state=2016)) - [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] - - ``random_state`` can also be any object with a method ``random`` that - returns floats between 0.0 and 1.0 (exclusive). - - >>> from random import Random - >>> randobj = Random(2016) - >>> list(random_sample(0.1, seq, random_state=randobj)) - [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98] - """ - ... - diff --git a/typings/toolz/recipes.pyi b/typings/toolz/recipes.pyi deleted file mode 100644 index 7723febc..00000000 --- a/typings/toolz/recipes.pyi +++ /dev/null @@ -1,42 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -__all__ = ('countby', 'partitionby') -def countby(key, seq): # -> dict[Unknown, int]: - """ Count elements of a collection by a key function - - >>> countby(len, ['cat', 'mouse', 'dog']) - {3: 2, 5: 1} - - >>> def iseven(x): return x % 2 == 0 - >>> countby(iseven, [1, 2, 3]) # doctest:+SKIP - {True: 1, False: 2} - - See Also: - groupby - """ - ... - -def partitionby(func, seq): # -> map[tuple[Unknown, ...]]: - """ Partition a sequence according to a function - - Partition `s` into a sequence of lists such that, when traversing - `s`, every time the output of `func` changes a new list is started - and that and subsequent items are collected into that list. - - >>> is_space = lambda c: c == " " - >>> list(partitionby(is_space, "I have space")) - [('I',), (' ',), ('h', 'a', 'v', 'e'), (' ',), ('s', 'p', 'a', 'c', 'e')] - - >>> is_large = lambda x: x > 10 - >>> list(partitionby(is_large, [1, 2, 1, 99, 88, 33, 99, -1, 5])) - [(1, 2, 1), (99, 88, 33, 99), (-1, 5)] - - See also: - partition - groupby - itertools.groupby - """ - ... - diff --git a/typings/toolz/sandbox/__init__.pyi b/typings/toolz/sandbox/__init__.pyi deleted file mode 100644 index c0c0a14b..00000000 --- a/typings/toolz/sandbox/__init__.pyi +++ /dev/null @@ -1,7 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -from .core import EqualityHashKey, unzip -from .parallel import fold - diff --git a/typings/toolz/sandbox/core.pyi b/typings/toolz/sandbox/core.pyi deleted file mode 100644 index 739e0775..00000000 --- a/typings/toolz/sandbox/core.pyi +++ /dev/null @@ -1,104 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -class EqualityHashKey: - """ Create a hash key that uses equality comparisons between items. - - This may be used to create hash keys for otherwise unhashable types: - - >>> from toolz import curry - >>> EqualityHashDefault = curry(EqualityHashKey, None) - >>> set(map(EqualityHashDefault, [[], (), [1], [1]])) # doctest: +SKIP - {=[]=, =()=, =[1]=} - - **Caution:** adding N ``EqualityHashKey`` items to a hash container - may require O(N**2) operations, not O(N) as for typical hashable types. - Therefore, a suitable key function such as ``tuple`` or ``frozenset`` - is usually preferred over using ``EqualityHashKey`` if possible. - - The ``key`` argument to ``EqualityHashKey`` should be a function or - index that returns a hashable object that effectively distinguishes - unequal items. This helps avoid the poor scaling that occurs when - using the default key. For example, the above example can be improved - by using a key function that distinguishes items by length or type: - - >>> EqualityHashLen = curry(EqualityHashKey, len) - >>> EqualityHashType = curry(EqualityHashKey, type) # this works too - >>> set(map(EqualityHashLen, [[], (), [1], [1]])) # doctest: +SKIP - {=[]=, =()=, =[1]=} - - ``EqualityHashKey`` is convenient to use when a suitable key function - is complicated or unavailable. For example, the following returns all - unique values based on equality: - - >>> from toolz import unique - >>> vals = [[], [], (), [1], [1], [2], {}, {}, {}] - >>> list(unique(vals, key=EqualityHashDefault)) - [[], (), [1], [2], {}] - - **Warning:** don't change the equality value of an item already in a hash - container. Unhashable types are unhashable for a reason. For example: - - >>> L1 = [1] ; L2 = [2] - >>> s = set(map(EqualityHashDefault, [L1, L2])) - >>> s # doctest: +SKIP - {=[1]=, =[2]=} - - >>> L1[0] = 2 # Don't do this! ``s`` now has duplicate items! - >>> s # doctest: +SKIP - {=[2]=, =[2]=} - - Although this may appear problematic, immutable data types is a common - idiom in functional programming, and``EqualityHashKey`` easily allows - the same idiom to be used by convention rather than strict requirement. - - See Also: - identity - """ - __slots__ = ... - _default_hashkey = ... - def __init__(self, key, item) -> None: - ... - - def __hash__(self) -> int: - ... - - def __eq__(self, other) -> bool: - ... - - def __ne__(self, other) -> bool: - ... - - def __str__(self) -> str: - ... - - def __repr__(self): # -> str: - ... - - - -def unzip(seq): # -> tuple[Unknown, ...] | tuple[map[Any] | Generator[tuple[Unknown | str, ...], None, None] | Generator[Unknown | str, None, None], ...]: - """Inverse of ``zip`` - - >>> a, b = unzip([('a', 1), ('b', 2)]) - >>> list(a) - ['a', 'b'] - >>> list(b) - [1, 2] - - Unlike the naive implementation ``def unzip(seq): zip(*seq)`` this - implementation can handle an infinite sequence ``seq``. - - Caveats: - - * The implementation uses ``tee``, and so can use a significant amount - of auxiliary storage if the resulting iterators are consumed at - different times. - - * The inner sequence cannot be infinite. In Python 3 ``zip(*seq)`` can be - used if ``seq`` is a finite sequence of infinite sequences. - - """ - ... - diff --git a/typings/toolz/sandbox/parallel.pyi b/typings/toolz/sandbox/parallel.pyi deleted file mode 100644 index 9c872462..00000000 --- a/typings/toolz/sandbox/parallel.pyi +++ /dev/null @@ -1,46 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -def fold(binop, seq, default=..., map=..., chunksize=..., combine=...): - """ - Reduce without guarantee of ordered reduction. - - inputs: - - ``binop`` - associative operator. The associative property allows us to - leverage a parallel map to perform reductions in parallel. - ``seq`` - a sequence to be aggregated - ``default`` - an identity element like 0 for ``add`` or 1 for mul - - ``map`` - an implementation of ``map``. This may be parallel and - determines how work is distributed. - ``chunksize`` - Number of elements of ``seq`` that should be handled - within a single function call - ``combine`` - Binary operator to combine two intermediate results. - If ``binop`` is of type (total, item) -> total - then ``combine`` is of type (total, total) -> total - Defaults to ``binop`` for common case of operators like add - - Fold chunks up the collection into blocks of size ``chunksize`` and then - feeds each of these to calls to ``reduce``. This work is distributed - with a call to ``map``, gathered back and then refolded to finish the - computation. In this way ``fold`` specifies only how to chunk up data but - leaves the distribution of this work to an externally provided ``map`` - function. This function can be sequential or rely on multithreading, - multiprocessing, or even distributed solutions. - - If ``map`` intends to serialize functions it should be prepared to accept - and serialize lambdas. Note that the standard ``pickle`` module fails - here. - - Example - ------- - - >>> # Provide a parallel map to accomplish a parallel sum - >>> from operator import add - >>> fold(add, [1, 2, 3, 4], chunksize=2, map=map) - 10 - """ - ... - diff --git a/typings/toolz/utils.pyi b/typings/toolz/utils.pyi deleted file mode 100644 index cb6a3b99..00000000 --- a/typings/toolz/utils.pyi +++ /dev/null @@ -1,8 +0,0 @@ -""" -This type stub file was generated by pyright. -""" - -def raises(err, lamda): # -> bool: - ... - -no_default = ... diff --git a/typings/vcr/__init__.pyi b/typings/vcr/__init__.pyi index ba5c4e76..7838a770 100644 --- a/typings/vcr/__init__.pyi +++ b/typings/vcr/__init__.pyi @@ -7,6 +7,9 @@ from .config import VCR from logging import NullHandler from .record_mode import RecordMode as mode +""" +This type stub file was generated by pyright. +""" __version__ = ... default_vcr = ... use_cassette = ... diff --git a/typings/vcr/_handle_coroutine.pyi b/typings/vcr/_handle_coroutine.pyi index 4e7d21f0..7f3ff6e2 100644 --- a/typings/vcr/_handle_coroutine.pyi +++ b/typings/vcr/_handle_coroutine.pyi @@ -2,6 +2,9 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" async def handle_coroutine(vcr, fn): ... diff --git a/typings/vcr/cassette.pyi b/typings/vcr/cassette.pyi index a791ad96..f505a781 100644 --- a/typings/vcr/cassette.pyi +++ b/typings/vcr/cassette.pyi @@ -4,6 +4,9 @@ This type stub file was generated by pyright. import wrapt +""" +This type stub file was generated by pyright. +""" log = ... class CassetteContextDecorator: """Context manager/decorator that handles installing the cassette and @@ -24,7 +27,7 @@ class CassetteContextDecorator: """ _non_cassette_arguments = ... @classmethod - def from_args(cls, cassette_class, **kwargs): # -> Self@CassetteContextDecorator: + def from_args(cls, cassette_class, **kwargs): ... def __init__(self, cls, args_getter) -> None: @@ -33,11 +36,11 @@ class CassetteContextDecorator: def __enter__(self): ... - def __exit__(self, *args): # -> None: + def __exit__(self, *args): ... @wrapt.decorator - def __call__(self, function, instance, args, kwargs): # -> Coroutine[Any, Any, Unknown] | Generator[Unknown, Unknown, None]: + def __call__(self, function, instance, args, kwargs): ... @staticmethod @@ -49,50 +52,50 @@ class CassetteContextDecorator: class Cassette: """A container for recorded requests and responses""" @classmethod - def load(cls, **kwargs): # -> Self@Cassette: + def load(cls, **kwargs): """Instantiate and load the cassette stored at the specified path.""" ... @classmethod - def use_arg_getter(cls, arg_getter): # -> CassetteContextDecorator: + def use_arg_getter(cls, arg_getter): ... @classmethod - def use(cls, **kwargs): # -> CassetteContextDecorator: + def use(cls, **kwargs): ... def __init__(self, path, serializer=..., persister=..., record_mode=..., match_on=..., before_record_request=..., before_record_response=..., custom_patches=..., inject=..., allow_playback_repeats=...) -> None: ... @property - def play_count(self): # -> int: + def play_count(self): ... @property - def all_played(self): # -> bool: + def all_played(self): """Returns True if all responses have been played, False otherwise.""" ... @property - def requests(self): # -> list[Unknown]: + def requests(self): ... @property - def responses(self): # -> list[Unknown]: + def responses(self): ... @property - def write_protected(self): # -> bool: + def write_protected(self): ... - def append(self, request, response): # -> None: + def append(self, request, response): """Add a request, response pair to this cassette""" ... def filter_request(self, request): ... - def can_play_response_for(self, request): # -> bool: + def can_play_response_for(self, request): ... def play_response(self, request): @@ -102,7 +105,7 @@ class Cassette: """ ... - def responses_of(self, request): # -> list[Unknown]: + def responses_of(self, request): """ Find the responses corresponding to a request. This function isn't actually used by VCR internally, but is @@ -110,10 +113,10 @@ class Cassette: """ ... - def rewind(self): # -> None: + def rewind(self): ... - def find_requests_with_most_matches(self, request): # -> list[Unknown]: + def find_requests_with_most_matches(self, request): """ Get the most similar request(s) stored in the cassette of a given request as a list of tuples like this: @@ -129,11 +132,11 @@ class Cassette: def __str__(self) -> str: ... - def __len__(self): # -> int: + def __len__(self): """Return the number of request,response pairs stored in here""" ... - def __contains__(self, request): # -> bool: + def __contains__(self, request): """Return whether or not a request has been stored""" ... diff --git a/typings/vcr/config.pyi b/typings/vcr/config.pyi index 032606f7..9a5f7ca0 100644 --- a/typings/vcr/config.pyi +++ b/typings/vcr/config.pyi @@ -2,34 +2,37 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" class VCR: @staticmethod - def is_test_method(method_name, function): # -> bool: + def is_test_method(method_name, function): ... @staticmethod - def ensure_suffix(suffix): # -> (path: Unknown) -> Unknown: + def ensure_suffix(suffix): ... def __init__(self, path_transformer=..., before_record_request=..., custom_patches=..., filter_query_parameters=..., ignore_hosts=..., record_mode=..., ignore_localhost=..., filter_headers=..., before_record_response=..., filter_post_data_parameters=..., match_on=..., before_record=..., inject_cassette=..., serializer=..., cassette_library_dir=..., func_path_generator=..., decode_compressed_response=...) -> None: ... - def use_cassette(self, path=..., **kwargs): # -> CassetteContextDecorator: + def use_cassette(self, path=..., **kwargs): ... - def get_merged_config(self, **kwargs): # -> dict[str, Module("vcr.config.yamlserializer") | Module("vcr.config.jsonserializer") | Type[FilesystemPersister] | Unknown | list[Unknown] | RecordMode | ((request: Unknown) -> Unknown) | ((response: Unknown) -> Unknown) | tuple[Unknown, ...] | bool | ((incoming: Unknown) -> Unknown) | ((function: Unknown) -> str) | None]: + def get_merged_config(self, **kwargs): ... - def register_serializer(self, name, serializer): # -> None: + def register_serializer(self, name, serializer): ... - def register_matcher(self, name, matcher): # -> None: + def register_matcher(self, name, matcher): ... - def register_persister(self, persister): # -> None: + def register_persister(self, persister): ... - def test_case(self, predicate=...): # -> type: + def test_case(self, predicate=...): ... diff --git a/typings/vcr/errors.pyi b/typings/vcr/errors.pyi index 2c0f6275..9c54c668 100644 --- a/typings/vcr/errors.pyi +++ b/typings/vcr/errors.pyi @@ -2,6 +2,9 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" class CannotOverwriteExistingCassetteException(Exception): def __init__(self, *args, **kwargs) -> None: ... diff --git a/typings/vcr/filters.pyi b/typings/vcr/filters.pyi index c1de01d5..a2b86bdd 100644 --- a/typings/vcr/filters.pyi +++ b/typings/vcr/filters.pyi @@ -2,6 +2,9 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" def replace_headers(request, replacements): """Replace headers in request according to replacements. The replacements should be a list of (key, value) pairs where the value can be any of: diff --git a/typings/vcr/matchers.pyi b/typings/vcr/matchers.pyi index 2991dc6a..0a588c6c 100644 --- a/typings/vcr/matchers.pyi +++ b/typings/vcr/matchers.pyi @@ -2,44 +2,47 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" log = ... -def method(r1, r2): # -> None: +def method(r1, r2): ... -def uri(r1, r2): # -> None: +def uri(r1, r2): ... -def host(r1, r2): # -> None: +def host(r1, r2): ... -def scheme(r1, r2): # -> None: +def scheme(r1, r2): ... -def port(r1, r2): # -> None: +def port(r1, r2): ... -def path(r1, r2): # -> None: +def path(r1, r2): ... -def query(r1, r2): # -> None: +def query(r1, r2): ... -def raw_body(r1, r2): # -> None: +def raw_body(r1, r2): ... -def body(r1, r2): # -> None: +def body(r1, r2): ... -def headers(r1, r2): # -> None: +def headers(r1, r2): ... _xml_header_checker = ... _xmlrpc_header_checker = ... _checker_transformer_pairs = ... -def requests_match(r1, r2, matchers): # -> bool: +def requests_match(r1, r2, matchers): ... -def get_matchers_results(r1, r2, matchers): # -> tuple[list[Unknown], list[Unknown]]: +def get_matchers_results(r1, r2, matchers): """ Get the comparison results of two requests as two list. The first returned list represents the matchers names that passed. diff --git a/typings/vcr/migration.pyi b/typings/vcr/migration.pyi index 9fff8d7f..f6b4d416 100644 --- a/typings/vcr/migration.pyi +++ b/typings/vcr/migration.pyi @@ -3,38 +3,28 @@ This type stub file was generated by pyright. """ """ -Migration script for old 'yaml' and 'json' cassettes - -.. warning:: Backup your cassettes files before migration. - -It merges and deletes the request obsolete keys (protocol, host, port, path) -into new 'uri' key. -Usage:: - - python -m vcr.migration PATH - -The PATH can be path to the directory with cassettes or cassette itself +This type stub file was generated by pyright. """ def preprocess_yaml(cassette): ... PARTS = ... -def build_uri(**parts): # -> LiteralString: +def build_uri(**parts): ... -def migrate_json(in_fp, out_fp): # -> bool: +def migrate_json(in_fp, out_fp): ... -def migrate_yml(in_fp, out_fp): # -> bool: +def migrate_yml(in_fp, out_fp): ... -def migrate(file_path, migration_fn): # -> bool: +def migrate(file_path, migration_fn): ... -def try_migrate(path): # -> bool: +def try_migrate(path): ... -def main(): # -> None: +def main(): ... if __name__ == "__main__": diff --git a/typings/vcr/patch.pyi b/typings/vcr/patch.pyi index cbc14389..36bef07d 100644 --- a/typings/vcr/patch.pyi +++ b/typings/vcr/patch.pyi @@ -7,7 +7,9 @@ import http.client as httplib import urllib3.connectionpool as cpool import requests.packages.urllib3.connectionpool as cpool -"""Utilities for patching in cassettes""" +""" +This type stub file was generated by pyright. +""" log = ... _HTTPConnection = httplib.HTTPConnection _HTTPSConnection = httplib.HTTPSConnection @@ -18,7 +20,7 @@ class CassettePatcherBuilder: def __init__(self, cassette) -> None: ... - def build(self): # -> chain[Any]: + def build(self): ... @@ -27,24 +29,24 @@ class ConnectionRemover: def __init__(self, connection_class) -> None: ... - def add_connection_to_pool_entry(self, pool, connection): # -> None: + def add_connection_to_pool_entry(self, pool, connection): ... - def remove_connection_to_pool_entry(self, pool, connection): # -> None: + def remove_connection_to_pool_entry(self, pool, connection): ... - def __enter__(self): # -> Self@ConnectionRemover: + def __enter__(self): ... - def __exit__(self, *args): # -> None: + def __exit__(self, *args): ... -def reset_patchers(): # -> Generator[_patch[_HTTPConnection] | _patch[_HTTPSConnection] | _patch[Unknown | VerifiedHTTPSConnection] | _patch[Unknown | HTTPConnection] | _patch[Unknown | HTTPSConnection] | _patch[Unknown] | _patch[_HTTPConnectionWithTimeout] | _patch[_HTTPSConnectionWithTimeout], Any, None]: +def reset_patchers(): ... @contextlib.contextmanager -def force_reset(): # -> Generator[None, Any, None]: +def force_reset(): ... diff --git a/typings/vcr/persisters/__init__.pyi b/typings/vcr/persisters/__init__.pyi index 006bc274..e897159a 100644 --- a/typings/vcr/persisters/__init__.pyi +++ b/typings/vcr/persisters/__init__.pyi @@ -2,3 +2,6 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" diff --git a/typings/vcr/record_mode.pyi b/typings/vcr/record_mode.pyi index 021601c2..7ff7ea05 100644 --- a/typings/vcr/record_mode.pyi +++ b/typings/vcr/record_mode.pyi @@ -4,6 +4,9 @@ This type stub file was generated by pyright. from enum import Enum +""" +This type stub file was generated by pyright. +""" class RecordMode(str, Enum): """ Configures when VCR will record to the cassette. diff --git a/typings/vcr/request.pyi b/typings/vcr/request.pyi index ffba4584..9fee2578 100644 --- a/typings/vcr/request.pyi +++ b/typings/vcr/request.pyi @@ -4,6 +4,9 @@ This type stub file was generated by pyright. from .util import CaseInsensitiveDict +""" +This type stub file was generated by pyright. +""" log = ... class Request: """ @@ -13,56 +16,56 @@ class Request: ... @property - def headers(self): # -> HeadersDict: + def headers(self): ... @headers.setter - def headers(self, value): # -> None: + def headers(self, value): ... @property - def body(self): # -> BytesIO | bytes: + def body(self): ... @body.setter - def body(self, value): # -> None: + def body(self, value): ... - def add_header(self, key, value): # -> None: + def add_header(self, key, value): ... @property - def scheme(self): # -> str: + def scheme(self): ... @property - def host(self): # -> str | None: + def host(self): ... @property - def port(self): # -> int | None: + def port(self): ... @property - def path(self): # -> str: + def path(self): ... @property - def query(self): # -> list[tuple[str, str]]: + def query(self): ... @property - def url(self): # -> Unknown: + def url(self): ... @property - def protocol(self): # -> str: + def protocol(self): ... def __str__(self) -> str: ... - def __repr__(self): # -> LiteralString: + def __repr__(self): ... @@ -86,7 +89,7 @@ class HeadersDict(CaseInsensitiveDict): For this reason, in cassettes I keep a dict with lists as keys, but once deserialized into VCR, I keep them as plain, naked dicts. """ - def __setitem__(self, key, value): # -> None: + def __setitem__(self, key, value): ... diff --git a/typings/vcr/serialize.pyi b/typings/vcr/serialize.pyi index 78f8ca65..58b2148d 100644 --- a/typings/vcr/serialize.pyi +++ b/typings/vcr/serialize.pyi @@ -2,8 +2,11 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" CASSETTE_FORMAT_VERSION = ... -def deserialize(cassette_string, serializer): # -> tuple[list[Request], list[Unknown]]: +def deserialize(cassette_string, serializer): ... def serialize(cassette_dict, serializer): diff --git a/typings/vcr/serializers/__init__.pyi b/typings/vcr/serializers/__init__.pyi index 006bc274..e897159a 100644 --- a/typings/vcr/serializers/__init__.pyi +++ b/typings/vcr/serializers/__init__.pyi @@ -2,3 +2,6 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" diff --git a/typings/vcr/stubs/__init__.pyi b/typings/vcr/stubs/__init__.pyi index c69b6fb5..53712b3f 100644 --- a/typings/vcr/stubs/__init__.pyi +++ b/typings/vcr/stubs/__init__.pyi @@ -9,7 +9,9 @@ from vcr.request import Request from vcr.errors import CannotOverwriteExistingCassetteException from . import compat -"""Stubs for patching HTTP and HTTPS requests""" +""" +This type stub file was generated by pyright. +""" log = ... class VCRFakeSocket: """ @@ -17,13 +19,13 @@ class VCRFakeSocket: Used when playing back cassettes, when there is no actual open socket. """ - def close(self): # -> None: + def close(self): ... - def settimeout(self, *args, **kwargs): # -> None: + def settimeout(self, *args, **kwargs): ... - def fileno(self): # -> Literal[0]: + def fileno(self): """ This is kinda crappy. requests will watch this descriptor and make sure it's not closed. @@ -33,14 +35,14 @@ class VCRFakeSocket: -def parse_headers(header_list): # -> HTTPMessage: +def parse_headers(header_list): """ Convert headers from our serialized dict with lists for keys to a HTTPMessage """ ... -def serialize_headers(response): # -> dict[Unknown, Unknown]: +def serialize_headers(response): ... class VCRHTTPResponse(HTTPResponse): @@ -51,66 +53,66 @@ class VCRHTTPResponse(HTTPResponse): ... @property - def closed(self): # -> bool: + def closed(self): ... - def read(self, *args, **kwargs): # -> bytes: + def read(self, *args, **kwargs): ... def readall(self): ... - def readinto(self, *args, **kwargs): # -> int: + def readinto(self, *args, **kwargs): ... - def readline(self, *args, **kwargs): # -> bytes: + def readline(self, *args, **kwargs): ... - def readlines(self, *args, **kwargs): # -> list[bytes]: + def readlines(self, *args, **kwargs): ... - def seekable(self): # -> bool: + def seekable(self): ... - def tell(self): # -> int: + def tell(self): ... - def isatty(self): # -> bool: + def isatty(self): ... - def seek(self, *args, **kwargs): # -> int: + def seek(self, *args, **kwargs): ... - def close(self): # -> Literal[True]: + def close(self): ... - def getcode(self): # -> int: + def getcode(self): ... - def isclosed(self): # -> bool: + def isclosed(self): ... - def info(self): # -> HTTPMessage: + def info(self): ... - def getheaders(self): # -> list[tuple[Unknown, Unknown]]: + def getheaders(self): ... - def getheader(self, header, default=...): # -> LiteralString | None: + def getheader(self, header, default=...): ... - def readable(self): # -> bool: + def readable(self): ... class VCRConnection: cassette = ... - def request(self, method, url, body=..., headers=..., *args, **kwargs): # -> None: + def request(self, method, url, body=..., headers=..., *args, **kwargs): """Persist the request metadata in self._vcr_request""" ... - def putrequest(self, method, url, *args, **kwargs): # -> None: + def putrequest(self, method, url, *args, **kwargs): """ httplib gives you more than one way to do it. This is a way to start building up a request. Usually followed by a bunch @@ -118,10 +120,10 @@ class VCRConnection: """ ... - def putheader(self, header, *values): # -> None: + def putheader(self, header, *values): ... - def send(self, data): # -> None: + def send(self, data): """ This method is called after request(), to add additional data to the body of the request. So if that happens, let's just append the data @@ -129,10 +131,10 @@ class VCRConnection: """ ... - def close(self): # -> None: + def close(self): ... - def endheaders(self, message_body=...): # -> None: + def endheaders(self, message_body=...): """ Normally, this would actually send the request to the server. We are not sending the request until getting the response, @@ -140,14 +142,14 @@ class VCRConnection: """ ... - def getresponse(self, _=..., **kwargs): # -> VCRHTTPResponse: + def getresponse(self, _=..., **kwargs): """Retrieve the response""" ... - def set_debuglevel(self, *args, **kwargs): # -> None: + def set_debuglevel(self, *args, **kwargs): ... - def connect(self, *args, **kwargs): # -> Any | None: + def connect(self, *args, **kwargs): """ httplib2 uses this. Connects to the server I'm assuming. @@ -157,17 +159,17 @@ class VCRConnection: ... @property - def sock(self): # -> Any | VCRFakeSocket | None: + def sock(self): ... @sock.setter - def sock(self, value): # -> None: + def sock(self, value): ... def __init__(self, *args, **kwargs) -> None: ... - def __setattr__(self, name, value): # -> None: + def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the VCRConnection need to be propagated to the real connection. @@ -181,7 +183,7 @@ class VCRConnection: """ ... - def __getattr__(self, name): # -> Any: + def __getattr__(self, name): """ Send requests for weird attributes up to the real connection (counterpart to __setattr above) diff --git a/typings/vcr/stubs/compat.pyi b/typings/vcr/stubs/compat.pyi index 95cde571..184fde1f 100644 --- a/typings/vcr/stubs/compat.pyi +++ b/typings/vcr/stubs/compat.pyi @@ -2,15 +2,18 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" def get_header(message, name): ... -def get_header_items(message): # -> Generator[tuple[Unknown, Unknown], Any, None]: +def get_header_items(message): ... -def get_headers(message): # -> Generator[tuple[Unknown, Unknown], Any, None]: +def get_headers(message): ... -def get_httpmessage(headers): # -> HTTPMessage: +def get_httpmessage(headers): ... diff --git a/typings/vcr/util.pyi b/typings/vcr/util.pyi index d887be34..a260a984 100644 --- a/typings/vcr/util.pyi +++ b/typings/vcr/util.pyi @@ -4,6 +4,9 @@ This type stub file was generated by pyright. from collections.abc import MutableMapping +""" +This type stub file was generated by pyright. +""" class CaseInsensitiveDict(MutableMapping): """ A case-insensitive ``dict``-like object. @@ -29,46 +32,46 @@ class CaseInsensitiveDict(MutableMapping): def __init__(self, data=..., **kwargs) -> None: ... - def __setitem__(self, key, value): # -> None: + def __setitem__(self, key, value): ... def __getitem__(self, key): ... - def __delitem__(self, key): # -> None: + def __delitem__(self, key): ... - def __iter__(self): # -> Generator[Unknown, None, None]: + def __iter__(self): ... - def __len__(self): # -> int: + def __len__(self): ... - def lower_items(self): # -> Generator[tuple[Unknown, Unknown], None, None]: + def lower_items(self): """Like iteritems(), but with all lowercase keys.""" ... def __eq__(self, other) -> bool: ... - def copy(self): # -> CaseInsensitiveDict: + def copy(self): ... - def __repr__(self): # -> str: + def __repr__(self): ... -def partition_dict(predicate, dictionary): # -> tuple[dict[Unknown, Unknown], dict[Unknown, Unknown]]: +def partition_dict(predicate, dictionary): ... -def compose(*functions): # -> (incoming: Unknown) -> Unknown: +def compose(*functions): ... def read_body(request): ... -def auto_decorate(decorator, predicate=...): # -> Type[DecorateAll]: +def auto_decorate(decorator, predicate=...): class DecorateAll(type): ...