|
12345678910111213141516171819202122232425 |
- #!/usr/bin/env bash
-
- set -euo pipefail
-
- export INCOMING="${INCOMING:-/data/incoming/}"
- export UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}"
-
- while [[ 1 ]] ; do
- # find every .warc.gz in the rsync directory
- find "${INCOMING}" -type f -not -name ".*"\
- | while read filename
- do
- # skip partial uploads
- if [[ $filename =~ rsync-tmp ]]
- then
- continue
- fi
- echo "Moving ${filename}"
- mkdir -vp "${UPLOAD_QUEUE}"
- echo mv -v "${filename}" "${UPLOAD_QUEUE}/"
- done
-
- echo "Sleeping 30 seconds..."
- sleep 30
- done
|