|
|
@@ -0,0 +1,61 @@ |
|
|
|
#!/bin/bash |
|
|
|
|
|
|
|
if [[ "$1" == '--test' ]] |
|
|
|
then |
|
|
|
## Self-test/example |
|
|
|
|
|
|
|
"$0" <<-EOF |
|
|
|
2022-11-28 19:23:10,000 - archivebot.pipeline.wpull_plugin - INFO - Ignore https://example.org/ignored using pattern /ignored$ |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetching ‘https://example.org/success-200’. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-200’: 200 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-204’: 204 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/success-304’: 304 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-401’: 401 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-403’: 403 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-404’: 404 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-405’: 405 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-410’: 410 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/ok-301’: 301 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/ok-dns-nxdomain’ encountered an error: DNS resolution failed: [Errno -2] Name or service not known |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/ok-dns-norecord’ encountered an error: DNS resolution failed: [Errno -5] No address associated with hostname |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429’: 429 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-418’: 418 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/error-dns’ encountered an error: DNS resolution error: All nameservers failed to answer the query ... |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 429 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-second-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry-with-redirect’: 429 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-429-successful-retry-with-redirect’: 302 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.base - ERROR - Fetching ‘https://example.org/error-dns-successful-retry’ encountered an error: DNS resolution failed: [Errno -2] Name or service not known |
|
|
|
2020-09-10 23:54:25,000 - wpull.processor.web - INFO - Fetched ‘https://example.org/error-dns-successful-retry’: 200 OK. Length: 1234 [text/html; charset=utf-8]. |
|
|
|
EOF |
|
|
|
exit |
|
|
|
fi |
|
|
|
|
|
|
|
patterns=( |
|
|
|
# 200s, 204s, 304s are green |
|
|
|
's,^\(.*’: \(200\|204\|304\) \),\x1b[0;32m\1,' |
|
|
|
|
|
|
|
# 301 and 302 are blue |
|
|
|
's,^\(.*’: 30[12] \),\x1b[0;36m\1,' |
|
|
|
|
|
|
|
# 401, 403, 404, 405, 410 are red |
|
|
|
's,^\(.*’: \(40[1345]\|410\) \),\x1b[0;31m\1,' |
|
|
|
|
|
|
|
# Any other response is bright red (yes, this adds unnecessary 'red' markers at the beginning of each line, which get overridden by the colours above) |
|
|
|
's,^\(.*’: \),\x1b[1;31m\1,' |
|
|
|
|
|
|
|
# As are any ERROR-level messages |
|
|
|
's,^\(.* - ERROR - \),\x1b[1;31m\1,' |
|
|
|
|
|
|
|
# Ignores (ArchiveBot-specific) are grey |
|
|
|
's,^\(.* - archivebot.pipeline.wpull_plugin - INFO - Ignore \),\x1b[0;90m\1,' |
|
|
|
|
|
|
|
# Reset colour at the end of line |
|
|
|
's,$,\x1b[0m,' |
|
|
|
) |
|
|
|
patterns="$(printf "; %s" "${patterns[@]}")" |
|
|
|
patterns="${patterns:2}" |
|
|
|
exec sed "${patterns}" |