diff --git a/kojismokydingo/__init__.py b/kojismokydingo/__init__.py index 3db17f4c..840c31d9 100644 --- a/kojismokydingo/__init__.py +++ b/kojismokydingo/__init__.py @@ -571,20 +571,20 @@ def bulk_load_tasks( :param results: mapping to store the results in. Default, produce a new dict - :raises NoSuchTask: if err is True and a task couldn'tb e loaded + :raises NoSuchTask: if err is True and a task couldn't be loaded :since: 1.0 """ results = {} if results is None else results + fn = partial(session.getTaskInfo, request=request, strict=False) - fn = partial(session.getTaskInfo, request=request) - - for key, info in iter_bulk_load(session, fn, task_ids, False, size): - if err and not info: - raise NoSuchTask(key) - else: - results[key] = info + for key_chunk in chunkseq(task_ids, size): + for key, info in zip(key_chunk, fn(key_chunk)): + if err and not info: + raise NoSuchTask(key) + else: + results[key] = info return results diff --git a/kojismokydingo/common.py b/kojismokydingo/common.py index cece3c8d..af180d0a 100644 --- a/kojismokydingo/common.py +++ b/kojismokydingo/common.py @@ -34,7 +34,7 @@ from fnmatch import fnmatchcase from functools import lru_cache from glob import glob -from itertools import filterfalse +from itertools import filterfalse, islice from operator import itemgetter from os.path import expanduser, isdir, join from typing import ( @@ -58,6 +58,7 @@ "find_config_files", "get_plugin_config", "globfilter", + "ichunkseq", "load_full_config", "load_plugin_config", "merge_extend", @@ -89,6 +90,38 @@ def chunkseq( offset in range(0, seqlen, chunksize)) +def ichunkseq( + seq: Iterable, + chunksize: int) -> Iterator[Iterable]: + """ + Similar to chunkseq, but lazy. Note that each chunk must be + exhausted before beginning a new chunk, as the chunks will be + reading from the original sequence only when they themselves are + iterated over. + + :param seq: a sequence to chunk up + + :param chunksize: max length for chunks + + :since: 2.1 + """ + + it = iter(seq) + cs = chunksize - 1 + + def chunk(primer): + yield primer + yield from islice(it, cs) + + while True: + try: + primer = next(it) + except StopIteration: + break + else: + yield chunk(primer) + + def escapable_replace( orig: str, character: str,