From 6773df8eae0091197d5a1bea73d03086ae5e0881 Mon Sep 17 00:00:00 2001 From: Katie Holly Date: Tue, 26 Mar 2019 00:48:21 +0100 Subject: [PATCH 1/3] suppress grep output --- offload-one | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offload-one b/offload-one index 002cea5..b7175b4 100755 --- a/offload-one +++ b/offload-one @@ -36,7 +36,7 @@ function mayicontinue { mayicontinue -if test -z "${OFFLOAD_TARGET}" && ! cat ./offload_targets 2> /dev/null | grep -E '^rsync://[^/]+/[^/]+'; then +if test -z "${OFFLOAD_TARGET}" && ! cat ./offload_targets 2> /dev/null | grep -qE '^rsync://[^/]+/[^/]+'; then echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file, aborting offload" sleep 30 exit 1 From 99f3db954062f80a513680963069fd237aa39b9f Mon Sep 17 00:00:00 2001 From: Katie Holly Date: Tue, 26 Mar 2019 00:53:18 +0100 Subject: [PATCH 2/3] Attempting to make rsync transaction more atomic --- offload-one | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offload-one b/offload-one index b7175b4..9c1c782 100755 --- a/offload-one +++ b/offload-one @@ -82,7 +82,7 @@ do continue fi echo "Offloading to ${_OFFLOAD_TARGET}/${ITEM}/" - rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${_OFFLOAD_TARGET}/${ITEM}/" + rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --delay-updates --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${_OFFLOAD_TARGET}/${ITEM}/" result=$? if [[ $result -ne 0 ]] then From 4d588ae2cd8113554ff1df9b512256a03f78cf25 Mon Sep 17 00:00:00 2001 From: Katie Holly Date: Tue, 26 Mar 2019 01:08:26 +0100 Subject: [PATCH 3/3] add config note for offload_targets file --- config.example.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.example.sh b/config.example.sh index 5b0869d..26be8ca 100755 --- a/config.example.sh +++ b/config.example.sh @@ -31,6 +31,8 @@ IA_ITEM_DATE=$( date +"%Y-%m" ) # offload items to another rsync storage instead of uploading to IA OFFLOAD_TARGET="rsync://somewhere-far-away:portnum/module-name/directory/" +# it is also possible to create a list of targets and the offloader will pick one at random and retry others on failure +# simply comment out the line above and put all rsync target urls separated by newline in a file called "offload_targets" ############### # DIRECTORIES #