Browse Source

Ignore book_op.php VirusCheck tasks

They're slow, and the data is already safely stored on both IA servers by the time only that task remains.
master
JustAnotherArchivist 1 year ago
parent
commit
b21cf13886
1 changed files with 1 additions and 1 deletions
  1. +1
    -1
      codearchiver-bot

+ 1
- 1
codearchiver-bot View File

@@ -282,7 +282,7 @@ function taint_block {

# Wait until all tasks for the item are done
while :; do
tasks="$(python3 -c 'import json, sys; o = json.load(sys.stdin); print(sum(o["value"]["summary"].values()))' < <({ curl --silent --verbose --fail --max-time 10 --header "Authorization: LOW ${IA_S3_ACCESS}:${IA_S3_SECRET}" "https://archive.org/services/tasks.php?identifier=${identifier}&summary=1&history=0" 2> >(log_loop 'curl IA tasks err: '); } | tee >(log_loop 'curl IA tasks out: ')))"
tasks="$(python3 -c 'import json, sys; o = json.load(sys.stdin); totalTasks = sum(o["value"]["summary"].values()); virusChecks = sum(1 for task in o["value"]["catalog"] if task.get("cmd") == "book_op.php" and task.get("args", {}).get("op10") == "VirusCheck"); print(f"Expected exactly 0 or 1 VirusCheck tasks, got {virusChecks}", file = sys.stderr) if virusChecks not in (0, 1) else None; print("Ignoring VirusCheck book_op task", file = sys.stderr) if virusChecks > 0 else None; print(totalTasks - min(virusChecks, 1))' < <({ curl --silent --verbose --fail --max-time 10 --header "Authorization: LOW ${IA_S3_ACCESS}:${IA_S3_SECRET}" "https://archive.org/services/tasks.php?identifier=${identifier}&catalog=1&history=0" 2> >(log_loop 'curl IA tasks err: '); } | tee >(log_loop 'curl IA tasks out: ')))"
if [[ "${tasks}" == '0' ]]; then
break
fi


Loading…
Cancel
Save