Parcourir la source
Merge pull request #12 from Fusl/fix-offloader
rsync empty files in offloader & skip non-existing tar files in uploader
master
Christopher Foo
il y a 5 ans
committed by
GitHub
Parent
révision
12fdba4837
Aucune clé connue n'a été trouvée dans la base pour cette signature
ID de la clé GPG: 4AEE18F83AFDEB23
2 fichiers modifiés avec
2 ajouts et
1 suppressions
-
offload-one
-
upload-one
|
|
@@ -65,7 +65,7 @@ echo "$( date ): Start offloading for item $ITEM" >> uploader.log |
|
|
|
result=1 |
|
|
|
while [[ $result -ne 0 ]] |
|
|
|
do |
|
|
|
rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/" |
|
|
|
rsync -r --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${OFFLOAD_TARGET}/${ITEM}/" |
|
|
|
result=$? |
|
|
|
if [[ $result -ne 0 ]] |
|
|
|
then |
|
|
|
|
|
@@ -67,6 +67,7 @@ size_hint=$( du --bytes -s "${UPLOADER_WORKING_DIR}/${ITEM}" | grep -oE "^[0-9]+ |
|
|
|
# (upload the large files first to optimise S3 snowballing) |
|
|
|
for ext in warc.gz tar json.gz |
|
|
|
do |
|
|
|
test "${ext}" == "tar" && ! test -f "${FILE_PREFIX}${ITEM}.megawarc.${ext}" && continue # skip non-existing tar files |
|
|
|
result=1 |
|
|
|
while [[ $result -ne 0 ]] |
|
|
|
do |
|
|
|