|
|
@@ -14,7 +14,7 @@ import typing |
|
|
|
import weakref |
|
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__) |
|
|
|
_logger = logging.getLogger(__name__) |
|
|
|
|
|
|
|
|
|
|
|
class InputURL: |
|
|
@@ -176,10 +176,10 @@ class HttpClient: |
|
|
|
for attempt in range(self._retries + 1): |
|
|
|
# The request is newly prepared on each retry because of potential cookie updates. |
|
|
|
req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers)) |
|
|
|
logger.info(f'Retrieving {req.url}') |
|
|
|
logger.debug(f'... with headers: {headers!r}') |
|
|
|
_logger.info(f'Retrieving {req.url}') |
|
|
|
_logger.debug(f'... with headers: {headers!r}') |
|
|
|
if data: |
|
|
|
logger.debug(f'... with data: {data!r}') |
|
|
|
_logger.debug(f'... with data: {data!r}') |
|
|
|
try: |
|
|
|
r = self._session.send(req, timeout = timeout) |
|
|
|
except requests.exceptions.RequestException as exc: |
|
|
@@ -189,7 +189,7 @@ class HttpClient: |
|
|
|
else: |
|
|
|
retrying = '' |
|
|
|
level = logging.ERROR |
|
|
|
logger.log(level, f'Error retrieving {req.url}: {exc!r}{retrying}') |
|
|
|
_logger.log(level, f'Error retrieving {req.url}: {exc!r}{retrying}') |
|
|
|
else: |
|
|
|
if responseOkCallback is not None: |
|
|
|
success, msg = responseOkCallback(r) |
|
|
@@ -198,7 +198,7 @@ class HttpClient: |
|
|
|
msg = f': {msg}' if msg else '' |
|
|
|
|
|
|
|
if success: |
|
|
|
logger.debug(f'{req.url} retrieved successfully{msg}') |
|
|
|
_logger.debug(f'{req.url} retrieved successfully{msg}') |
|
|
|
return r |
|
|
|
else: |
|
|
|
if attempt < self._retries: |
|
|
@@ -207,14 +207,14 @@ class HttpClient: |
|
|
|
else: |
|
|
|
retrying = '' |
|
|
|
level = logging.ERROR |
|
|
|
logger.log(level, f'Error retrieving {req.url}{msg}{retrying}') |
|
|
|
_logger.log(level, f'Error retrieving {req.url}{msg}{retrying}') |
|
|
|
if attempt < self._retries: |
|
|
|
sleepTime = 1.0 * 2**attempt # exponential backoff: sleep 1 second after first attempt, 2 after second, 4 after third, etc. |
|
|
|
logger.info(f'Waiting {sleepTime:.0f} seconds') |
|
|
|
_logger.info(f'Waiting {sleepTime:.0f} seconds') |
|
|
|
time.sleep(sleepTime) |
|
|
|
else: |
|
|
|
msg = f'{self._retries + 1} requests to {req.url} failed, giving up.' |
|
|
|
logger.fatal(msg) |
|
|
|
_logger.fatal(msg) |
|
|
|
raise HttpError(msg) |
|
|
|
raise RuntimeError('Reached unreachable code') |
|
|
|
|
|
|
@@ -240,9 +240,9 @@ class ModuleMeta(type): |
|
|
|
if class_.name in cls.__modulesByName: |
|
|
|
raise RuntimeError(f'Class name collision: {class_.name!r} is already known') |
|
|
|
cls.__modulesByName[class_.name] = weakref.ref(class_) |
|
|
|
logger.info(f'Found {class_.name!r} module {class_.__module__}.{class_.__name__}') |
|
|
|
_logger.info(f'Found {class_.name!r} module {class_.__module__}.{class_.__name__}') |
|
|
|
else: |
|
|
|
logger.info(f'Found nameless module {class_.__module__}.{class_.__name__}') |
|
|
|
_logger.info(f'Found nameless module {class_.__module__}.{class_.__name__}') |
|
|
|
return class_ |
|
|
|
|
|
|
|
@classmethod |
|
|
@@ -252,7 +252,7 @@ class ModuleMeta(type): |
|
|
|
if classRef := cls.__modulesByName.get(name): |
|
|
|
class_ = classRef() |
|
|
|
if class_ is None: |
|
|
|
logger.info(f'Module {name!r} is gone, dropping') |
|
|
|
_logger.info(f'Module {name!r} is gone, dropping') |
|
|
|
del cls.__modulesByName[name] |
|
|
|
return class_ |
|
|
|
|
|
|
@@ -263,7 +263,7 @@ class ModuleMeta(type): |
|
|
|
# Housekeeping first: remove dead modules |
|
|
|
for name in list(cls.__modulesByName): # create a copy of the names list so the dict can be modified in the loop |
|
|
|
if cls.__modulesByName[name]() is None: |
|
|
|
logger.info(f'Module {name!r} is gone, dropping') |
|
|
|
_logger.info(f'Module {name!r} is gone, dropping') |
|
|
|
del cls.__modulesByName[name] |
|
|
|
|
|
|
|
for name, classRef in cls.__modulesByName.items(): |
|
|
@@ -284,11 +284,11 @@ class ModuleMeta(type): |
|
|
|
|
|
|
|
if module.name is not None and module.name in cls.__modulesByName: |
|
|
|
del cls.__modulesByName[module.name] |
|
|
|
logger.info(f'Module {module.name!r} dropped') |
|
|
|
_logger.info(f'Module {module.name!r} dropped') |
|
|
|
|
|
|
|
def __del__(self, *args, **kwargs): |
|
|
|
if self.name is not None and self.name in type(self).__modulesByName: |
|
|
|
logger.info(f'Module {self.name!r} is being destroyed, dropping') |
|
|
|
_logger.info(f'Module {self.name!r} is being destroyed, dropping') |
|
|
|
del type(self).__modulesByName[self.name] |
|
|
|
# type has no __del__ method, no need to call it. |
|
|
|
|
|
|
@@ -328,7 +328,7 @@ def get_module_class(inputUrl: InputURL) -> typing.Type[Module]: |
|
|
|
# Check if the URL references one of the modules directly |
|
|
|
if inputUrl.moduleScheme: |
|
|
|
if module := ModuleMeta.get_module_by_name(inputUrl.moduleScheme): |
|
|
|
logger.info(f'Selecting module {module.__module__}.{module.__name__}') |
|
|
|
_logger.info(f'Selecting module {module.__module__}.{module.__name__}') |
|
|
|
return module |
|
|
|
else: |
|
|
|
raise RuntimeError(f'No module with name {inputUrl.moduleScheme!r} exists') |
|
|
@@ -336,11 +336,11 @@ def get_module_class(inputUrl: InputURL) -> typing.Type[Module]: |
|
|
|
# Check if exactly one of the modules matches |
|
|
|
matches = [class_ for class_ in ModuleMeta.iter_modules() if class_.matches(inputUrl)] |
|
|
|
if len(matches) >= 2: |
|
|
|
logger.error('Multiple matching modules for input URL') |
|
|
|
logger.debug(f'Matching modules: {matches!r}') |
|
|
|
_logger.error('Multiple matching modules for input URL') |
|
|
|
_logger.debug(f'Matching modules: {matches!r}') |
|
|
|
raise RuntimeError('Multiple matching modules for input URL') |
|
|
|
if matches: |
|
|
|
logger.info(f'Selecting module {matches[0].__module__}.{matches[0].__name__}') |
|
|
|
_logger.info(f'Selecting module {matches[0].__module__}.{matches[0].__name__}') |
|
|
|
return matches[0] |
|
|
|
raise RuntimeError('No matching modules for input URL') |
|
|
|
|
|
|
|