Browse Source

Filter out lines with invalid UTF-8

master
JustAnotherArchivist 10 months ago
parent
commit
a85ffe791b
1 changed files with 3 additions and 0 deletions
  1. +3
    -0
      html-extract-stupid

+ 3
- 0
html-extract-stupid View File

@@ -34,6 +34,9 @@
# Filter out lines without an attribute value
grep -Pva '^[a-zA-Z]+ $' |

# Remove lines with invalid UTF-8
LANG=C.UTF-8 grep -a '^.*$' |

# img srcset splitting
python3 -c 'import os, re, sys'$'\n''try:'$'\n'' for l in map(str.strip, sys.stdin):'$'\n'' tag, value = l.split(" ", 1)'$'\n'' tag = tag.lower()'$'\n'' if tag != "imgsrcset":'$'\n'' print(l); continue'$'\n'' for url in re.split(r"\s+\d+[wx]\s*(?:,\s*|$)|,\s+", value.strip()):'$'\n'' if url: print(f"img {url}")'$'\n''except BrokenPipeError:'$'\n'' os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()); sys.exit(1)' |



Loading…
Cancel
Save