From e5055f79e7120591c4ce413c926db2517ea9d622 Mon Sep 17 00:00:00 2001
From: Katie Holly <git@meo.ws>
Date: Mon, 25 Mar 2019 15:03:21 +0100
Subject: [PATCH] rsync empty files in offloader & skip non-existing tar files
 in uploader

Fixes #11
---
 offload-one | 2 +-
 upload-one  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/offload-one b/offload-one
index 5ad40f4..4d229bc 100755
--- a/offload-one
+++ b/offload-one
@@ -65,7 +65,7 @@ echo "$( date ): Start offloading for item $ITEM" >> uploader.log
 result=1
 while [[ $result -ne 0 ]]
 do
-  rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/"
+  rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/"
   result=$?
   if [[ $result -ne 0 ]]
   then
diff --git a/upload-one b/upload-one
index c62cd92..2d77982 100755
--- a/upload-one
+++ b/upload-one
@@ -67,6 +67,7 @@ size_hint=$( du --bytes -s "${UPLOADER_WORKING_DIR}/${ITEM}" | grep -oE "^[0-9]+
 # (upload the large files first to optimise S3 snowballing)
 for ext in warc.gz tar json.gz
 do
+  test "${ext}" == "tar" && ! test -f "${FILE_PREFIX}${ITEM}.megawarc.${ext}" && continue # skip non-existing tar files
   result=1
   while [[ $result -ne 0 ]]
   do