From 002c1eb7aeb1c617030fbb6bbfc41f7047681264 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Tue, 11 Jan 2022 22:26:05 +0000 Subject: [PATCH] Wait until item exists IA doesn't immediately create the item on CreateMultipartUpload, so if it didn't already exist, UploadPart would fail for a while and we'd waste bandwidth. --- ia-upload-stream | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ia-upload-stream b/ia-upload-stream index f7c5717..9f76d94 100755 --- a/ia-upload-stream +++ b/ia-upload-stream @@ -212,6 +212,19 @@ def upload(item, filename, metadata, *, iaConfigFile = None, partSize = 100*1024 uploadId = m[1] logger.info(f'Got upload ID {uploadId}') + # Wait for the item to exist; if the above created the item, it takes a little while for IA to actually create the bucket, and uploads would fail with a 404 until then. + for attempt in range(1, tries + 1): + logger.info(f'Checking for existence of {item}') + r = requests.get(f'https://s3.us.archive.org/{item}/', headers = headers) + if r.status_code == 200: + break + sleepTime = min(3 ** attempt, 30) + retrying = f', retrying after {sleepTime} seconds' if attempt < tries else '' + logger.error(f'Got status code {r.status_code} from IA S3 on checking for item existence{retrying}') + if attempt == tries: + raise UploadError('Item still does not exist', r = r, uploadId = uploadId, parts = parts) + time.sleep(sleepTime) + # Upload the data in parts if parts is None: parts = []