#!/bin/bash if [[ "$1" == '--test' ]] then ## Self-test/example "$0" <<-EOF 2022-11-28 19:23:10,000 - archivebot.pipeline.wpull_plugin - INFO - Ignore https://example.org/ignored using pattern /ignored$ 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetching ‘https://example.org/success-200’. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-200’: 200 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-204’: 204 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-304’: 304 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-401’: 401 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-403’: 403 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-404’: 404 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-405’: 405 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-410’: 410 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-301’: 301 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/ok-dns-nxdomain’ encountered an error: DNS resolution failed: [Errno -2] Name or service not known 2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/ok-dns-norecord’ encountered an error: DNS resolution failed: [Errno -5] No address associated with hostname 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429’: 429 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-418’: 418 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/error-dns’ encountered an error: DNS resolution error: All nameservers failed to answer the query ... 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry-with-redirect’: 429 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry-with-redirect’: 302 OK. Length: 1234 [text/html; charset=utf-8]. 2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/error-dns-successful-retry’ encountered an error: DNS resolution failed: [Errno -2] Name or service not known 2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-dns-successful-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. EOF exit fi patterns=( # 200s, 204s, 304s are green 's,^\(.*’: \(200\|204\|304\) \),\x1b[0;32m\1,' # 301 and 302 are blue 's,^\(.*’: 30[12] \),\x1b[0;36m\1,' # 401, 403, 404, 405, 410 are red 's,^\(.*’: \(40[1345]\|410\) \),\x1b[0;31m\1,' # Any other response is bright red (yes, this adds unnecessary 'red' markers at the beginning of each line, which get overridden by the colours above) 's,^\(.*’: \),\x1b[1;31m\1,' # As are any ERROR-level messages 's,^\(.* - ERROR - \),\x1b[1;31m\1,' # Ignores (ArchiveBot-specific) are grey 's,^\(.* - archivebot.pipeline.wpull_plugin - INFO - Ignore \),\x1b[0;90m\1,' # Reset colour at the end of line 's,$,\x1b[0m,' ) patterns="$(printf "; %s" "${patterns[@]}")" patterns="${patterns:2}" exec sed "${patterns}"