|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131 |
- #!/bin/bash
- # Offloads megawarcs from the upload queue.
- # (Needs a config.sh in the working directory.)
- #
- # ./offload-one
- #
- # 1. Grabs an item from UPLOAD_QUEUE_DIR
- # 2. Reserves the item by moving the directory to the
- # UPLOADER_WORKING_DIR
- # 3. Offloads the item to the target defined in OFFLOAD_TARGET
- # or in the offload_targets file
- # 4. Removes the source files from the working directory
- # If COMPLETED_DIR is set, offloaded files are moved there.
- #
- # The program exits with 1 on any nontransient error.
- #
-
- SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
- source ./config.sh || exit 1
-
- mkdir -p "$UPLOAD_QUEUE_DIR" || exit 1
- mkdir -p "$UPLOADER_WORKING_DIR" || exit 1
-
- if [ ! -z "$COMPLETED_DIR" ]
- then
- mkdir -p "$COMPLETED_DIR" || exit 1
- fi
-
- function mayicontinue {
- echo
- # echo "May I continue?"
- # read
- # echo
- }
-
- mayicontinue
-
- if test -z "${OFFLOAD_TARGET}" && ! cat ./offload_targets 2> /dev/null | grep -qE '^rsync://[^/]+/[^/]+'; then
- echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file, aborting offload"
- sleep 30
- exit 1
- fi
-
- # try to grab an item from UPLOAD_QUEUE_DIR
- ITEM=none
- while [[ $ITEM = none ]]
- do
- possible_item=$( ls -1 "$UPLOAD_QUEUE_DIR" | grep -E '[0-9]{14}_[a-f0-9]{8}$' | sort | head -n 1 )
- if test -n "${possible_item}"
- then
- echo "Trying to grab $possible_item"
- if mv "$UPLOAD_QUEUE_DIR/$possible_item" "$UPLOADER_WORKING_DIR/"
- then
- ITEM=$possible_item
- else
- echo "Failed to move $possible_item"
- sleep 5
- fi
- else
- date
- echo "No current item found!"
- sleep 30
- exit 0
- fi
- done
-
-
- echo "$( date ): Start offloading for item $ITEM" >> uploader.log
-
- result=1
- while [[ $result -ne 0 ]]
- do
- _OFFLOAD_TARGET="${OFFLOAD_TARGET}"
- if test -z "${_OFFLOAD_TARGET}"; then
- _OFFLOAD_TARGET=$(cat "./offload_targets" 2> /dev/null | grep -E '^rsync://[^/]+/[^/]+' | shuf -n 1)
- fi
- if test -z "${_OFFLOAD_TARGET}"; then
- echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file"
- echo "Will retry in 30 seconds"
- sleep 30
- continue
- fi
- echo "Offloading to ${_OFFLOAD_TARGET}/${ITEM}/"
- rsync -rltv --timeout=900 --contimeout=60 --sockopts=SO_SNDBUF=8388608,SO_RCVBUF=8388608 --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --delay-updates --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${_OFFLOAD_TARGET}/${ITEM}/"
- result=$?
- if [[ $result -ne 0 ]]
- then
- date
- echo "Error while offloading $ITEM, rsync said $result"
- echo "Will retry in 30 seconds"
- sleep 30
- fi
- done
-
- echo "Offloaded $ITEM"
-
- echo "$( date ): Completed offloading for item $ITEM" >> uploader.log
-
-
- mayicontinue
-
-
- # move or remove megawarc
- if [ -z "$COMPLETED_DIR" ]
- then
- # remove
- rm -rf "${UPLOADER_WORKING_DIR}/${ITEM}"
- result=$?
-
- if [[ $result -ne 0 ]]
- then
- date
- echo "rm -rf megawarc exited with $result for $ITEM"
- exit 1
- fi
- else
- # move
- mv "${UPLOADER_WORKING_DIR}/${ITEM}" "${COMPLETED_DIR}/"
- result=$?
-
- if [[ $result -ne 0 ]]
- then
- date
- echo "rm -rf megawarc exited with $result for $ITEM"
- exit 1
- fi
- fi
-
- exit 0
-
|