ab-offload/mover.sh

#!/usr/bin/env bash

set -euo pipefail

export INCOMING="${INCOMING:-/data/incoming/}"
export UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}"

while [[ 1 ]] ; do
	# find every .warc.gz in the rsync directory
	find "${INCOMING}" -type f -not -name ".*"\
	| while read filename
	do
		# skip partial uploads
		if [[ $filename =~ rsync-tmp ]]
		then
			continue
		fi
		echo "Moving ${filename}"
		mkdir -vp "${UPLOAD_QUEUE}"
		echo mv -v "${filename}" "${UPLOAD_QUEUE}/"
	done

	echo "Sleeping 30 seconds..."
	sleep 30
done