Browse Source

Record wrapper script in meta WARC as well

master
JustAnotherArchivist 3 years ago
parent
commit
5fe595d71c
1 changed files with 9 additions and 1 deletions
  1. +9
    -1
      s3-bucket-list-qwarc

+ 9
- 1
s3-bucket-list-qwarc View File

@@ -90,6 +90,11 @@ while [[ "${prefix}" == */ ]]; do prefix="${prefix%/}"; done # Remove trailing s
prefix="${prefix//\//_}" # Replace slashes with underscores

# Ensure no collisions
if [[ -e s3-bucket-list-qwarc ]]
then
echo 'Error: s3-bucket-list-qwarc exists in this directory.' >&2
exit 1
fi
if [[ "$(shopt -s nullglob; echo "${prefix}"*)" ]]
then
echo "Error: there already exist files with the prefix '${prefix}' in this directory." >&2
@@ -223,7 +228,7 @@ readarray code <<EOF


specDependencies = qwarc.utils.SpecDependencies(
files = [os.environ['S3_MARKERS_FILENAME']],
files = ['s3-bucket-list-qwarc', os.environ['S3_MARKERS_FILENAME']],
extra = {'env': {k: os.environ.get(k) for k in ('S3BL_CMD', 'S3_FORMAT', 'S3_BUCKET_URL', 'S3_MARKERS_FILENAME', 'S3_WITH_LIST_URLS')}}
)
EOF
@@ -232,6 +237,9 @@ printf '%s' "${code[@]# }" >"${prefix}.py" # That's a tab character after the h
# Generate the markers file
{ if [[ -z "${noStartMarker}" ]]; then echo; fi; printf '%s\n' "$@"; if [[ -z "${noEndMarker}" ]]; then echo; fi; } >"${prefix}-markers"

# Copy this script
rsync -a "$0" s3-bucket-list-qwarc

# Collect environment variables
envvars=()
envvars+=(S3BL_CMD="${cmd}")


Loading…
Cancel
Save