diff --git a/ia-upload-stream b/ia-upload-stream index f7c5717..9f76d94 100755 --- a/ia-upload-stream +++ b/ia-upload-stream @@ -212,6 +212,19 @@ def upload(item, filename, metadata, *, iaConfigFile = None, partSize = 100*1024 uploadId = m[1] logger.info(f'Got upload ID {uploadId}') + # Wait for the item to exist; if the above created the item, it takes a little while for IA to actually create the bucket, and uploads would fail with a 404 until then. + for attempt in range(1, tries + 1): + logger.info(f'Checking for existence of {item}') + r = requests.get(f'https://s3.us.archive.org/{item}/', headers = headers) + if r.status_code == 200: + break + sleepTime = min(3 ** attempt, 30) + retrying = f', retrying after {sleepTime} seconds' if attempt < tries else '' + logger.error(f'Got status code {r.status_code} from IA S3 on checking for item existence{retrying}') + if attempt == tries: + raise UploadError('Item still does not exist', r = r, uploadId = uploadId, parts = parts) + time.sleep(sleepTime) + # Upload the data in parts if parts is None: parts = []