diff --git a/README.md b/README.md index ccfcf2b..09e29d2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Stuff I frequently use in various unrelated projects that deal with web requests. +## Functions + +### gather_in_batches + +Makes it more convenient to run awaitable objects in concurrent batches. + ## Decorators ### in_async_session diff --git a/setup.cfg b/setup.cfg index 7f2cfc1..a10c77e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = webutils-df -version = 0.0.2 +version = 0.0.3 author = Daniil F. author_email = mail@placeholder123.to description = Miscellaneous web utilities diff --git a/src/webutils/util.py b/src/webutils/util.py index 5fb4240..9b6e6e6 100644 --- a/src/webutils/util.py +++ b/src/webutils/util.py @@ -1,7 +1,8 @@ import logging +import asyncio from functools import wraps from inspect import signature -from typing import Callable, Dict, Tuple, Any +from typing import Callable, Awaitable, Dict, Tuple, Any from aiohttp.client import ClientSession @@ -83,3 +84,32 @@ def in_async_session(_func: Callable = None, *, logger.debug("Temporary `ClientSession` closed") return wrapper return decorator if _func is None else decorator(_func) + + +async def gather_in_batches(batch_size: int, *aws: Awaitable, return_exceptions: bool = False) -> list: + """ + Simple extension of the `asyncio.gather` function to make it easy to run awaitable objects in concurrent batches. + (see: https://docs.python.org/3/library/asyncio-task.html#asyncio.gather) + + A batch is run concurrently using `gather`, while the calls to `gather` for each batch are done sequentially. + This can be useful if for example there is a very large number of requests to a single website to be made + and you want to make use of concurrency, but not by performing all of them at the same time. + + Args: + batch_size: + The maximum number of awaitable objects to run concurrently at any given moment. + If this is higher than the number of awaitable objects, this is equivalent to a single `gather` call. + Every batch will have this number of concurrent runs, but the last batch can obviously be smaller. + aws: + The awaitable objects/coroutines; subsets of these are passed to `gather` always maintaining overall order + return_exceptions (optional): + Passed into each `gather` call + + Returns: + The aggregate list of returned values for all awaitable objects. + The order of result values corresponds to the order of `aws`. + """ + results = [] + for idx in range(0, len(aws), batch_size): + results.extend(await asyncio.gather(*aws[idx:idx + batch_size], return_exceptions=return_exceptions)) + return results