|
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273 |
- #!/bin/bash
- set -e
-
- if [[ $# -ne 1 || "$1" == '-h' || "$1" == '--help' ]]
- then
- echo 'Usage: transfer.notkiska.pw-check-ia TKPPATH'
- echo 'TKPPATH is the path of a file on transfer.notkiska.pw, e.g. "123F3V/twitter-#qanon"'
- echo 'Checks that the file is archived correctly on IA by downloading both copies and comparing the SHA-1.'
- echo 'If the TKPCHECK_CACHE_DIR environment variable is set, it is used as a cache for the CDXs to avoid redownloading them from IA on every check.'
- exit 1
- fi
-
- if [[ "${TKPCHECK_CACHE_DIR}" ]]
- then
- if [[ -e "${TKPCHECK_CACHE_DIR}" && ! -d "${TKPCHECK_CACHE_DIR}" ]]
- then
- echo "Error: ${TKPCHECK_CACHE_DIR} is not a directory." >&2
- exit 1
- fi
- fi
-
- file="$1"
- fileid="${file%/*}"
- filename="${file#*/}"
-
- echo "Downloading from transfer.notkiska.pw" >&2
- tkphash="$(curl "https://transfer.notkiska.pw/${file}" | sha1sum | tee /dev/fd/2)"
-
- echo "Retrieving WARC list from viewer" >&2
- mapfile -t warcs < <(curl -s "https://archive.fart.website/archivebot/viewer/api/v1/search.json?q=urls-transfer.notkiska.pw-transfer.notkiska.pw-" | python3 -c 'import json,sys; [print(x["job_id"]) for x in json.loads(sys.stdin.read())["results"]]' | sed 's,^,https://archive.fart.website/archivebot/viewer/job/,' | xargs curl -s | grep -Po 'href="\Khttps://archive.org/download/[^/"]+/[^/"]+-\d\d\d\d\d\.warc\.gz(?=")')
-
- cdxLines=()
- for warc in "${warcs[@]}"
- do
- cdx="${warc::-3}.os.cdx.gz"
- mapfile -t -O ${#cdxLines[@]} cdxLines < <(
- {
- if [[ "${TKPCHECK_CACHE_DIR}" ]]
- then
- cdxfn="${cdx:29}"
- mkdir -p "${TKPCHECK_CACHE_DIR}/${cdxfn%/*}"
- if [[ ! -e "${TKPCHECK_CACHE_DIR}/${cdxfn}" ]]
- then
- echo "Fetching ${cdx} into local cache" >&2
- curl -L "${cdx}" >"${TKPCHECK_CACHE_DIR}/${cdxfn}"
- fi
- cat "${TKPCHECK_CACHE_DIR}/${cdxfn}"
- else
- echo "Fetching ${cdx}" >&2
- curl -L "${cdx}"
- fi
- } | zgrep -F "/${fileid}/" 2>/dev/null | tee /dev/fd/2
- )
- done
-
- if [[ ${#cdxLines[@]} -ne 1 ]]
- then
- echo "Not exactly one matching CDX line found, cannot continue" >&2
- exit 1
- fi
-
- read -r length offset iapath < <(awk '{print $9 " " $10 " " $11}' <<<"${cdxLines[0]}")
- echo "Fetching ${offset}-$((${offset}+${length})) from IA ${iapath}" >&2
- iahash="$(curl -L --range "${offset}-$((${offset}+${length}))" "https://archive.org/download/${iapath}" | zcat 2>/dev/null | awk '/^\r$/ {empty+=1; next} (empty >= 2)' | sha1sum | tee /dev/fd/2)"
-
- if [[ "${tkphash}" == "${iahash}" ]]
- then
- echo OK >&2
- exit 0
- else
- echo "Hash mismatch!" >&2
- exit 1
- fi
|