Add uploader.

Dockerfile

@@ -18,6 +18,7 @@ WORKDIR /
 COPY mover.sh /mover.sh
 COPY entrypoint.sh /entrypoint.sh
 COPY uploader.sh /uploader.sh
+COPY upload-single.sh /upload-single.sh
 #ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt
 #ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py
 

upload-single.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ ! -d "$1" ]] ; then
+	echo "Target directory $1 does not exist!"
+	exit
+fi
+
+set -x
+
+cd "$1"
+upload="$(basename "$(pwd)")"
+export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" )
+export YEAR=`date +%Y`
+echo ia upload "archiveteam_archivebot_go_${upload}" * \
+	--sleep=120 \
+	--retries=100 \
+	--metadata="noarchivetorrent:true" \
+	--metadata="size-hint:${CUR_SIZE}" \
+	--metadata="mediatype:web" \
+	--metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \
+	--metadata="title: Archiveteam: Archivebot GO Pack ${upload}" \
+	--metadata="collection:archivebot" \
+	--metadata="creator:Archive Team" \
+	--metadata="date:${YEAR}" \
+	--checksum \
+	--delete
+
+# TEST SLEEP
+sleep 6000
+
+cd ..
+echo rmdir "$upload"

uploader.sh

@@ -1,2 +1,16 @@
-#!/bin/bash
-set -exuo pipefail
+#!/usr/bin/env bash
+set -euo pipefail
+
+export UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}"
+mkdir -p "${UPLOAD_QUEUE}"
+
+cd "${UPLOAD_QUEUE}"
+while [ ! -f ../STOP ]
+do
+	echo "Starting new loop..."
+	for pack in */; do
+		echo "Trying claim ${pack}..."
+		flock --verbose -x -nb "$pack" /upload-single.sh "$pack" || echo "Could not claim pack!"
+	done
+	sleep 30
+done