|
|
@@ -59,7 +59,7 @@ class Item: |
|
|
|
out.append((key, value)) |
|
|
|
return out |
|
|
|
|
|
|
|
async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True): |
|
|
|
async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True, timeout = 60): |
|
|
|
''' |
|
|
|
HTTP GET or POST a URL |
|
|
|
|
|
|
@@ -71,6 +71,7 @@ class Item: |
|
|
|
To remove one of the default headers, pass a value of None. |
|
|
|
If a header appears multiple times, only the last one is used. To send a header multiple times, pass a tuple of values. |
|
|
|
verify_ssl: bool, whether the SSL/TLS certificate should be validated |
|
|
|
timeout: int or float, how long the fetch may take at most in total (sending request until finishing reading the response) |
|
|
|
|
|
|
|
Returns response (a ClientResponse object or None) and history (a tuple of (response, exception) tuples). |
|
|
|
response can be None and history can be an empty tuple, depending on the circumstances (e.g. timeouts). |
|
|
@@ -92,7 +93,7 @@ class Item: |
|
|
|
writeToWarc = True |
|
|
|
try: |
|
|
|
try: |
|
|
|
with _aiohttp.Timeout(60): |
|
|
|
with _aiohttp.Timeout(timeout): |
|
|
|
self.logger.info(f'Fetching {url}') |
|
|
|
response = await self.session.request(method, url, data = data, headers = headers, allow_redirects = False, verify_ssl = verify_ssl) |
|
|
|
try: |
|
|
|