You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
- #!/usr/bin/env bash
-
- set -euo pipefail
-
- export INCOMING="${INCOMING:-/data/incoming/}"
- export UPLOAD_QUEUE="${UPLOAD_QUEUE:-/data/upload-queue/}"
-
- while [[ 1 ]] ; do
- # find every .warc.gz in the rsync directory
- find "${INCOMING}" -type f -not -name ".*"\
- | while read filename
- do
- # skip partial uploads
- if [[ $filename =~ rsync-tmp ]]
- then
- continue
- fi
- echo "Moving ${filename}"
- mkdir -vp "${UPLOAD_QUEUE}"
- mv -v "${filename}" "${UPLOAD_QUEUE}/"
- done
-
- echo "Sleeping 30 seconds..."
- sleep 30
- done
|