A framework for quick web archiving
Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

233 Zeilen
8.0 KiB

  1. from qwarc.const import *
  2. import aiohttp
  3. import asyncio
  4. import functools
  5. import logging
  6. import os
  7. import pkg_resources
  8. import platform
  9. import time
  10. PAGESIZE = os.sysconf('SC_PAGE_SIZE')
  11. def get_rss():
  12. '''Get the current RSS of this process in bytes'''
  13. with open('/proc/self/statm', 'r') as fp:
  14. return int(fp.readline().split()[1]) * PAGESIZE
  15. def get_disk_free():
  16. '''Get the current free disk space on the relevant partition in bytes'''
  17. st = os.statvfs('.')
  18. return st.f_bavail * st.f_frsize
  19. def uses_too_much_memory(limit):
  20. '''
  21. Check whether the process is using too much memory
  22. For performance reasons, this actually only checks the memory usage on every 100th call.
  23. '''
  24. uses_too_much_memory.callCounter += 1
  25. # Only check every hundredth call
  26. if uses_too_much_memory.callCounter % 100 == 0 and get_rss() > limit:
  27. return True
  28. return False
  29. uses_too_much_memory.callCounter = 0
  30. def too_little_disk_space(limit):
  31. '''
  32. Check whether the disk space is too small
  33. For performance reasons, this actually only checks the free disk space on every 100th call.
  34. '''
  35. too_little_disk_space.callCounter += 1
  36. if too_little_disk_space.callCounter % 100 == 0:
  37. too_little_disk_space.currentResult = (get_disk_free() < limit)
  38. return too_little_disk_space.currentResult
  39. too_little_disk_space.callCounter = 0
  40. too_little_disk_space.currentResult = False
  41. # https://stackoverflow.com/a/4665027
  42. def find_all(aStr, sub):
  43. '''Generator yielding the start positions of every non-overlapping occurrence of sub in aStr.'''
  44. start = 0
  45. while True:
  46. start = aStr.find(sub, start)
  47. if start == -1:
  48. return
  49. yield start
  50. start += len(sub)
  51. def str_get_between(aStr, a, b):
  52. '''Get the string after the first occurrence of a in aStr and the first occurrence of b after that of a, or None if there is no such string.'''
  53. aPos = aStr.find(a)
  54. if aPos == -1:
  55. return None
  56. offset = aPos + len(a)
  57. bPos = aStr.find(b, offset)
  58. if bPos == -1:
  59. return None
  60. return aStr[offset:bPos]
  61. def maybe_str_get_between(x, a, b):
  62. '''Like str_get_between, but returns None if x evaluates to False and converts it to a str before matching.'''
  63. if x:
  64. return str_get_between(str(x), a, b)
  65. def str_get_all_between(aStr, a, b):
  66. '''Generator yielding every string between occurrences of a in aStr and the following occurrence of b.'''
  67. #TODO: This produces half-overlapping matches: str_get_all_between('aabc', 'a', 'c') will yield 'ab' and 'b'.
  68. # Might need to implement sending an offset to the find_all generator to work around this, or discard aOffset values which are smaller than the previous bPos+len(b).
  69. for aOffset in find_all(aStr, a):
  70. offset = aOffset + len(a)
  71. bPos = aStr.find(b, offset)
  72. if bPos != -1:
  73. yield aStr[offset:bPos]
  74. def maybe_str_get_all_between(x, a, b):
  75. '''Like str_get_all_between, but yields no elements if x evaluates to False and converts x to a str before matching.'''
  76. if x:
  77. yield from str_get_all_between(str(x), a, b)
  78. def generate_range_items(start, stop, step):
  79. '''
  80. Generator for items of `step` size between `start` and `stop` (inclusive)
  81. Yields strings of the form `'a-b'` where `a` and `b` are integers such that `b - a + 1 == step`, `min(a) == start`, and `max(b) == stop`.
  82. `b - a + 1` may be unequal to `step` on the last item if `(stop - start + 1) % step != 0` (see examples below).
  83. Note that `a` and `b` can be equal on the last item if `(stop - start) % step == 0` (see examples below).
  84. Examples:
  85. - generate_range_items(0, 99, 10) yields '0-9', '10-19', '20-29', ..., '90-99'
  86. - generate_range_items(0, 42, 10): '0-9', '10-19', '20-29', '30-39', '40-42'
  87. - generate_range_items(0, 20, 10): '0-9', '10-19', '20-20'
  88. '''
  89. for i in range(start, stop + 1, step):
  90. yield f'{i}-{min(i + step - 1, stop)}'
  91. async def handle_response_default(url, attempt, response, exc):
  92. '''
  93. The default response handler, which behaves as follows:
  94. - If there is no response (e.g. timeout error), retry the retrieval after a delay of 5 seconds.
  95. - If the response has any of the status codes 401, 403, 404, 405, or 410, treat it as a permanent error and return.
  96. - If there was any exception and it is a asyncio.TimeoutError or a aiohttp.ClientError, treat as a potentially temporary error and retry the retrieval after a delay of 5 seconds.
  97. - If the response has any of the status codes 200, 204, 206, or 304, treat it as a success and return.
  98. - If the response has any of the status codes 301, 302, 303, 307, or 308, follow the redirect target if specified or return otherwise.
  99. - Otherwise, treat as a potentially temporary error and retry the retrieval after a delay of 5 seconds.
  100. - All responses are written to WARC by default.
  101. Note that this handler does not limit the number of retries on errors.
  102. Parameters: url (yarl.URL instance), attempt (int), response (aiohttp.ClientResponse or None), exc (Exception or None)
  103. At least one of response and exc is not None.
  104. Returns: (one of the qwarc.RESPONSE_* constants, bool signifying whether to write to WARC or not)
  105. '''
  106. #TODO: Document that `attempt` is reset on redirects
  107. if response is None:
  108. await asyncio.sleep(5)
  109. return ACTION_RETRY, True
  110. if response.status in (401, 403, 404, 405, 410):
  111. return ACTION_IGNORE, True
  112. if exc is not None and isinstance(exc, (asyncio.TimeoutError, aiohttp.ClientError)):
  113. await asyncio.sleep(5)
  114. return ACTION_RETRY, True
  115. if response.status in (200, 204, 206, 304):
  116. return ACTION_SUCCESS, True
  117. if response.status in (301, 302, 303, 307, 308):
  118. return ACTION_FOLLOW_OR_SUCCESS, True
  119. await asyncio.sleep(5)
  120. return ACTION_RETRY, True
  121. async def handle_response_ignore_redirects(url, attempt, response, exc):
  122. '''A response handler that does not follow redirects, i.e. treats them as a success instead. It behaves as handle_response_default otherwise.'''
  123. action, writeToWarc = await handle_response_default(url, attempt, response, exc)
  124. if action == ACTION_FOLLOW_OR_SUCCESS:
  125. action = ACTION_SUCCESS
  126. return action, writeToWarc
  127. def handle_response_limit_error_retries(maxRetries, handler = handle_response_default):
  128. '''A response handler that limits the number of retries on errors. It behaves as handler otherwise, which defaults to handle_response_default.
  129. Technically, this is actually a response handler factory. This is so that the intuitive use works: fetch(..., responseHandler = handle_response_limit_error_retries(5))
  130. If you use the same limit many times, you should keep the return value (the response handler) of this method and reuse it to avoid creating a new function every time.
  131. '''
  132. async def _handler(url, attempt, response, exc):
  133. action, writeToWarc = await handler(url, attempt, response, exc)
  134. if action == ACTION_RETRY and attempt > maxRetries:
  135. action = ACTION_RETRIES_EXCEEDED
  136. return action, writeToWarc
  137. return _handler
  138. def _get_dependency_versions(pkg):
  139. pending = {pkg}
  140. have = {pkg}
  141. while pending:
  142. key = pending.pop()
  143. try:
  144. dist = pkg_resources.get_distribution(key)
  145. except pkg_resources.DistributionNotFound:
  146. logging.error(f'Unable to get distribution {key}')
  147. yield dist.key, dist.version
  148. for requirement in dist.requires():
  149. if requirement.key not in have:
  150. pending.add(requirement.key)
  151. have.add(requirement.key)
  152. @functools.lru_cache(maxsize = 1)
  153. def get_software_info():
  154. # Taken from crocoite.utils, authored by PromyLOPh in commit 6ccd72ab on 2018-12-08 under MIT licence
  155. return {
  156. 'platform': platform.platform(),
  157. 'python': {
  158. 'implementation': platform.python_implementation(),
  159. 'version': platform.python_version(),
  160. 'build': platform.python_build(),
  161. },
  162. 'self': [{"package": package, "version": version} for package, version in _get_dependency_versions(__package__)],
  163. }
  164. class LogFormatter(logging.Formatter):
  165. def __init__(self):
  166. super().__init__('%(asctime)s.%(msecs)03dZ %(levelname)s %(itemString)s %(message)s', datefmt = '%Y-%m-%d %H:%M:%S')
  167. self.converter = time.gmtime
  168. def format(self, record):
  169. if not hasattr(record, 'itemString'):
  170. if hasattr(record, 'itemType') and hasattr(record, 'itemValue'):
  171. record.itemString = f'{record.itemType}:{record.itemValue}'
  172. else:
  173. record.itemString = 'None'
  174. return super().format(record)