From 1010769c3cebf27681c7210b17ff1a3df0425fdd Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Mon, 29 Nov 2021 15:05:48 +0000 Subject: [PATCH] Handle connection errors --- ia-upload-stream | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ia-upload-stream b/ia-upload-stream index ae71ee1..230ae90 100755 --- a/ia-upload-stream +++ b/ia-upload-stream @@ -113,14 +113,19 @@ def upload(item, filename, metadata, *, iaConfigFile = None, partSize = 4, tries for attempt in range(1, tries + 1): if attempt > 1: logger.info(f'Retrying part {partNumber}') - r = requests.put(f'{url}?partNumber={partNumber}&uploadId={uploadId}', headers = {**headers, 'Content-MD5': contentMd5}, data = data) - if r.status_code == 200: - break + try: + r = requests.put(f'{url}?partNumber={partNumber}&uploadId={uploadId}', headers = {**headers, 'Content-MD5': contentMd5}, data = data) + except (ConnectionError, requests.exceptions.RequestException) as e: + err = f'error {type(e).__module__}.{type(e).__name__} {e!s}' + else: + if r.status_code == 200: + break + err = f'status {r.status_code}' sleepTime = min(3 ** attempt, 30) retrying = f', retrying after {sleepTime} seconds' if attempt < tries else '' - logger.error(f'Got status {r.status_code} from IA S3 on uploading part {partNumber}{retrying}') + logger.error(f'Got {err} from IA S3 on uploading part {partNumber}{retrying}') if attempt == tries: - raise UploadError(f'Got status {r.status_code} from IA S3 on uploading part {partNumber}', r = r, uploadId = uploadId, parts = parts) + raise UploadError(f'Got {err} from IA S3 on uploading part {partNumber}', r = r, uploadId = uploadId, parts = parts) time.sleep(sleepTime) logger.info(f'Upload OK, ETag: {r.headers["ETag"]}') parts.append((partNumber, r.headers['ETag']))