Skip to content

Commit

Permalink
replace aiofiles with async pathlib where possible
Browse files Browse the repository at this point in the history
  • Loading branch information
monosans committed Jan 7, 2025
1 parent e2e95df commit 07148d8
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 19 deletions.
18 changes: 8 additions & 10 deletions proxy_scraper_checker/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import asyncio
import logging
import sys
from pathlib import Path
from typing import TYPE_CHECKING

import aiofiles
import rich
from aiohttp import ClientSession, TCPConnector
from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn
Expand Down Expand Up @@ -78,12 +78,6 @@ def get_async_run() -> Callable[[Coroutine[Any, Any, T]], T]:
return asyncio.run


async def read_config(file: str, /) -> dict[str, Any]:
async with aiofiles.open(file, "rb") as f:
content = await f.read()
return tomllib.loads(utils.bytes_decode(content))


def get_summary_table(
*, before: Mapping[ProxyType, int], after: Mapping[ProxyType, int]
) -> Table:
Expand All @@ -102,8 +96,12 @@ def get_summary_table(


async def main() -> None:
cfg = await read_config("config.toml")
if cfg["debug"]:
config = tomllib.loads(
utils.bytes_decode(
await asyncio.to_thread(Path("config.toml").read_bytes)
)
)
if config["debug"]:
logging.root.setLevel(logging.DEBUG)
should_save = False
try:
Expand All @@ -114,7 +112,7 @@ async def main() -> None:
raise_for_status=True,
fallback_charset_resolver=http.fallback_charset_resolver,
) as session:
settings = await Settings.from_mapping(cfg, session=session)
settings = await Settings.from_mapping(config, session=session)
storage = ProxyStorage(protocols=settings.sources)
with Progress(
TextColumn("[yellow]{task.fields[module]}"),
Expand Down
8 changes: 2 additions & 6 deletions proxy_scraper_checker/geodb.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ async def _read_etag() -> str | None:
await asyncio.to_thread(
fs.add_permission, GEODB_ETAG_PATH, stat.S_IRUSR
)
async with aiofiles.open(GEODB_ETAG_PATH, "rb") as etag_file:
content = await etag_file.read()
content = await asyncio.to_thread(GEODB_ETAG_PATH.read_bytes)
except FileNotFoundError:
return None
return bytes_decode(content)
Expand All @@ -42,10 +41,7 @@ async def _save_etag(etag: str, /) -> None:
await asyncio.to_thread(
fs.add_permission, GEODB_ETAG_PATH, stat.S_IWUSR, missing_ok=True
)
async with aiofiles.open(
GEODB_ETAG_PATH, "w", encoding="utf-8"
) as etag_file:
await etag_file.write(etag)
await asyncio.to_thread(GEODB_ETAG_PATH.write_text, etag, encoding="utf-8")


async def _save_geodb(
Expand Down
5 changes: 2 additions & 3 deletions proxy_scraper_checker/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import asyncio
import itertools
import logging
from pathlib import Path
from typing import TYPE_CHECKING

import aiofiles
from aiohttp import ClientResponseError, ClientTimeout
from aiohttp_socks import ProxyType

Expand Down Expand Up @@ -42,8 +42,7 @@ async def scrape_one(
content = await response.read()
text = get_response_text(response=response, content=content)
else:
async with aiofiles.open(source, "rb") as f:
content = await f.read()
content = await asyncio.to_thread(Path(source).read_bytes)
text = bytes_decode(content)
except ClientResponseError as e:
_logger.warning(
Expand Down

0 comments on commit 07148d8

Please sign in to comment.