Browse Source

Merge pull request #12 from Fusl/fix-offloader

rsync empty files in offloader & skip non-existing tar files in uploader
master
Christopher Foo 5 years ago
committed by GitHub
parent
commit
12fdba4837
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 2 additions and 1 deletions
  1. +1
    -1
      offload-one
  2. +1
    -0
      upload-one

+ 1
- 1
offload-one View File

@@ -65,7 +65,7 @@ echo "$( date ): Start offloading for item $ITEM" >> uploader.log
result=1
while [[ $result -ne 0 ]]
do
rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/"
rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/"
result=$?
if [[ $result -ne 0 ]]
then


+ 1
- 0
upload-one View File

@@ -67,6 +67,7 @@ size_hint=$( du --bytes -s "${UPLOADER_WORKING_DIR}/${ITEM}" | grep -oE "^[0-9]+
# (upload the large files first to optimise S3 snowballing)
for ext in warc.gz tar json.gz
do
test "${ext}" == "tar" && ! test -f "${FILE_PREFIX}${ITEM}.megawarc.${ext}" && continue # skip non-existing tar files
result=1
while [[ $result -ne 0 ]]
do


Loading…
Cancel
Save