little-things/dir-to-ia

#!/bin/bash
# Requires Bash 4.3+
if [[ "$1" == '-h' || "$1" == '--help' ]]; then
	printf 'Usage: %q\n' "$0"
	printf 'Uploads files from each subdirectory of the PWD into an IA item using the dirname as the identifier\n'
	printf 'Configuration happens through a subdir/.dir-to-ia.config file, a Bash script that gets sourced to get the config values\n'
	printf "Use '%q --example-config' to get an example configuration with explanations\n" "$0"
	printf 'The upload log for each item gets written to subdir/.dir-to-ia.log\n'
	exit 1
fi

# Default config values
sha256=no
rm=no
rmwait=yes
clobber=no
derive=no
iaconfigfile=

if [[ "$1" == '--example-config' ]]; then
	cat <<-EOF
		# Calculate SHA-256 hash of each file after uploading
		sha256=${sha256}

		# Remove local file after successful upload
		rm=${rm}

		# Delay removal until IA processed the upload
		rmwait=${rmwait}

		# Clobber existing files in IA item (no = existing copy is moved to history/files/ by IA)
		clobber=${clobber}

		# Queue derive after upload (applied after every file!)
		derive=${derive}

		# 'ia' config file (default: empty, meaning to use ia's own default paths and precedence rules)
		iaconfigfile=${iaconfigfile}

		# Item metadata (array with 'key:value' elements); the only mandatory variable with no default
		metadata=('collection:opensource' "date:$(printf '%(%Y-%m-%d)T')")
	EOF
	exit
fi

function tsprintf {
	datefmt='%(%Y-%m-%d %H:%M:%S)T';
	if [[ $# -eq 1 ]]; then
		printf "${datefmt} %s\n" -1 "$@";
	else
		# First argument is the format, rest are arguments to printf; pass through `while read` loop to have a better chance of line buffering.
		{
			printf '%(%Y-%m-%d %H:%M:%S)T ';
			printf "$1" "${@:2}";
			printf '\n';
		} | while IFS= read -r line; do printf '%s\n' "${line}"; done
	fi
}

if [[ "${DEBUG}" ]]; then
	function dbgprint { for l; do tsprintf "[DEBUG] ${l}" >&2; done; }
else
	function dbgprint { :; }
fi

for cmd in ia-upload-stream ia-wait-item-tasks ia; do
	if ! command -v "${cmd}" &>/dev/null; then
		echo "Error: ${cmd} not found, make sure it is in PATH" >&2
		exit 1
	fi
done

while :; do
	for dir in */; do
		dir="${dir%/}"
		identifier="${dir}"
		if [[ ! "${dir}" =~ ^[a-zA-Z0-9] ]]; then
			continue
		fi
		if [[ ! -e "${dir}/.dir-to-ia.config" ]]; then
			continue
		fi

		dbgprint "Processing ${dir}"

		# Everything from here on is executed in a subshell so that the config sourcing can't affect other items.
		# It would be possible to do that without a subshell, but this is easier.
		(
			# Source and check configuration
			. "${dir}/.dir-to-ia.config" || { tsprintf "Sourcing ${dir}/.dir-to-ia.config failed" >&2; exit 1; }
			configbroken=
			for v in sha256 rm rmwait clobber derive; do
				if [[ "${!v}" != 'yes' && "${!v}" != 'no' ]]; then
					tsprintf "Error in %q: %s must be 'yes' or 'no'" "${dir}/.dir-to-ia.config" "${v}" >&2
					configbroken=yes
				fi
			done
			if read -r _ attrs _ < <(declare -p metadata 2>/dev/null); [[ "${attrs}" != *a* ]]; then
				tsprintf 'Error in %q: metadata missing or not an array' "${dir}/.dir-to-ia.config" >&2
				configbroken=yes
			else
				for f in "${metadata[@]}"; do
					if [[ "${f}" != *:* ]]; then
						tsprintf 'Error in %q: metadata field missing colon: %s' "${dir}/.dir-to-ia.config" "${f}" >&2
						configbroken=yes
					fi
				done
			fi
			if [[ "${configbroken}" ]]; then
				exit 1
			fi
			if [[ "${rm}" == 'no' && "${clobber}" == 'yes' ]]; then
				tsprintf 'Error in %q: rm=no and clobber=yes is not permitted' "${dir}/.dir-to-ia.config" >&2
				exit 1
			fi
			dbgprint 'Configuration:' \
			         "  sha256=${sha256}" \
			         "  rm=${rm}" \
			         "  rmwait=${rmwait}" \
			         "  clobber=${clobber}" \
			         "  derive=${derive}" \
			         "  iaconfigfile${iaconfigfile:+: }${iaconfigfile:- not set}" \
			         "  metadata=($(printf %q "${metadata[0]}")$(if [[ ${#metadata[@]} -gt 1 ]]; then printf ' %q' "${metadata[@]:1}"; fi))"

			# If removing local files is disabled, check first which ones are already on IA so they can be skipped.
			#TODO Do this only if there are files to upload
			if [[ "${rm}" == 'no' ]]; then
				dbgprint 'Retrieving existing files on IA...'
				#TODO Figure out a better way to verify correct retrieval than appending a placeholder.
				readarray -d $'\0' -t iafiles < <(curl --silent --location --max-time 10 --fail "https://archive.org/metadata/${identifier}" | python3 -c 'import json, sys; o = json.load(sys.stdin); {print(f["name"], end = "\0") for f in o.get("files", []) if not f["name"].startswith(sys.argv[1])}; print("__dir-to-ia_end__", end = "\0")' "${identifier}")
				if [[ "${#iafiles[@]}" -eq 0 || "${iafiles[-1]}" != '__dir-to-ia_end__' ]]; then
					tsprintf 'Error: could not retrieve metadata' >&2
					exit 1
				fi
				unset iafiles[-1]
				dbgprint "Existing files on IA:$(printf " %q" "${iafiles[@]}")"
			fi

			# Loop over local files and upload them
			while IFS= read -r -d $'\0' fn; do
				dbgprint "Considering file ${fn}"
				if [[ "${fn}" == .dir-to-ia.* ]]; then
					dbgprint "${fn} is a dir-to-ia file, skipping"
					continue
				fi
				if [[ ! -f "${dir}/${fn}" ]]; then
					# Should never happen since the `find` command already uses `-type f`
					dbgprint "${fn} is not a regular file, skipping"
					continue
				fi
				if [[ "${fn}" == "${identifier}"* ]]; then
					dbgprint "${fn} starts with the identifier, skipping"
					continue
				fi
				if [[ "${rm}" == 'no' ]]; then
					found=
					for remoteFn in "${iafiles[@]}"; do
						if [[ "${fn}" == "${remoteFn}" ]]; then
							found=yes
							break
						fi
					done
					if [[ "${found}" ]]; then
						dbgprint "${fn} found in IA item, skipping"
						continue
					fi
				fi

				tsprintf 'Uploading %q to %q...' "${fn}" "${identifier}" >&2
				cmd=('ia-upload-stream')
				if [[ "${clobber}" == 'yes' ]]; then cmd+=('--clobber'); fi
				if [[ "${derive}" == 'no' ]]; then cmd+=('--no-derive'); fi
				if [[ "${iaconfigfile}" ]]; then cmd+=('--ia-config-file' "${iaconfigfile}"); fi
				cmd+=("${identifier}")
				cmd+=("${fn}")
				cmd+=("${metadata[@]}")

				dbgprint "Upload command:$(printf " %q" "${cmd[@]}")"
				"${cmd[@]}" <"${dir}/${fn}" || { tsprintf "ia-upload-stream exited with status $?" >&2; exit 1; }

				if [[ "${sha256}" == 'yes' ]]; then
					tsprintf 'Calculating SHA-256...' >&2
					(cd "${dir}" && sha256sum "${fn}") || { tsprintf "sha256sum exited with status $?" >&2; exit 1; }
				fi

				if [[ "${rm}" == 'yes' ]]; then
					if [[ "${rmwait}" == 'yes' ]]; then
						tsprintf 'Waiting for IA to process the upload...' >&2
						ia-wait-item-tasks "${identifier}" || { tsprintf "ia-wait-item-tasks exited with status $?" >&2; exit 1; }
					fi
					tsprintf 'IA upload processing finished, removing %q' "${fn}" >&2
					# No option to run an extra SHA-1 check or similar since ia-upload-stream already sends an MD5 for each chunk, so corruption should be impossible.
					rm "${dir}/${fn}"
				fi
			done < <(cd "${dir}" && find . -type f -print0 | sed -z 's,^\./,,')
		) &> >(tee -a "${dir}/.dir-to-ia.log" >&2)
		dbgprint "Done with ${dir}"
	done
	sleep 60
done