@@ -18,6 +18,7 @@ WORKDIR / | |||||
COPY mover.sh /mover.sh | COPY mover.sh /mover.sh | ||||
COPY entrypoint.sh /entrypoint.sh | COPY entrypoint.sh /entrypoint.sh | ||||
COPY uploader.sh /uploader.sh | COPY uploader.sh /uploader.sh | ||||
COPY upload-single.sh /upload-single.sh | |||||
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt | #ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt | ||||
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py | #ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py | ||||
@@ -0,0 +1,33 @@ | |||||
#!/bin/bash | |||||
set -euo pipefail | |||||
if [[ ! -d "$1" ]] ; then | |||||
echo "Target directory $1 does not exist!" | |||||
exit | |||||
fi | |||||
set -x | |||||
cd "$1" | |||||
upload="$(basename "$(pwd)")" | |||||
export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" ) | |||||
export YEAR=`date +%Y` | |||||
echo ia upload "archiveteam_archivebot_go_${upload}" * \ | |||||
--sleep=120 \ | |||||
--retries=100 \ | |||||
--metadata="noarchivetorrent:true" \ | |||||
--metadata="size-hint:${CUR_SIZE}" \ | |||||
--metadata="mediatype:web" \ | |||||
--metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \ | |||||
--metadata="title: Archiveteam: Archivebot GO Pack ${upload}" \ | |||||
--metadata="collection:archivebot" \ | |||||
--metadata="creator:Archive Team" \ | |||||
--metadata="date:${YEAR}" \ | |||||
--checksum \ | |||||
--delete | |||||
# TEST SLEEP | |||||
sleep 6000 | |||||
cd .. | |||||
echo rmdir "$upload" |
@@ -1,2 +1,16 @@ | |||||
#!/bin/bash | |||||
set -exuo pipefail | |||||
#!/usr/bin/env bash | |||||
set -euo pipefail | |||||
export UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}" | |||||
mkdir -p "${UPLOAD_QUEUE}" | |||||
cd "${UPLOAD_QUEUE}" | |||||
while [ ! -f ../STOP ] | |||||
do | |||||
echo "Starting new loop..." | |||||
for pack in */; do | |||||
echo "Trying claim ${pack}..." | |||||
flock --verbose -x -nb "$pack" /upload-single.sh "$pack" || echo "Could not claim pack!" | |||||
done | |||||
sleep 30 | |||||
done |