ab-offload/mover.sh

#!/usr/bin/env bash

set -euo pipefail

INCOMING="${INCOMING:-/data/incoming/}"
UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}"

# find every .warc.gz in the rsync directory
find "${INCOMING}" -type f -not -name ".*"\
| while read filename
do
  # skip partial uploads
  if [[ $filename =~ rsync-tmp ]]
  then
    continue
  fi

  echo "Moving ${filename}"
  mkdir -vp "${UPLOAD_QUEUE}"
  mv -v "${filename}" "${UPLOAD_QUEUE}/"
done