From a85e80ffa26bf9ac4d056c3349fdf3f941a34fe7 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Sat, 11 Jul 2020 01:11:38 +0000 Subject: [PATCH] Configurable request timeout --- qwarc/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qwarc/__init__.py b/qwarc/__init__.py index 55b26d6..261bd14 100644 --- a/qwarc/__init__.py +++ b/qwarc/__init__.py @@ -59,7 +59,7 @@ class Item: out.append((key, value)) return out - async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True): + async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True, timeout = 60): ''' HTTP GET or POST a URL @@ -71,6 +71,7 @@ class Item: To remove one of the default headers, pass a value of None. If a header appears multiple times, only the last one is used. To send a header multiple times, pass a tuple of values. verify_ssl: bool, whether the SSL/TLS certificate should be validated + timeout: int or float, how long the fetch may take at most in total (sending request until finishing reading the response) Returns response (a ClientResponse object or None) and history (a tuple of (response, exception) tuples). response can be None and history can be an empty tuple, depending on the circumstances (e.g. timeouts). @@ -92,7 +93,7 @@ class Item: writeToWarc = True try: try: - with _aiohttp.Timeout(60): + with _aiohttp.Timeout(timeout): self.logger.info(f'Fetching {url}') response = await self.session.request(method, url, data = data, headers = headers, allow_redirects = False, verify_ssl = verify_ssl) try: