JustAnotherArchivist
/
little-things
2
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
606 Commits
1 Branch
2.9 MiB
 
 
 
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
little-things/dir-to-ia

445 lines
19 KiB
Raw Permalink Blame History 

  1. #!/usr/bin/env bash

  2. printf '\e[1;33mWARNING: This script is experimental and is not intended for productive use yet. Use at your own risk.\e[0m\n\n' >&2

  3. 

  4. # Requires Bash 4.3+

  5. if [[ "$1" == '-h' || "$1" == '--help' ]]; then

  6. 	printf 'Usage: %q\n' "$0"

  7. 	printf '\n'

  8. 	printf 'Uploads files from each subdirectory of the PWD into an IA item using the dirname as the identifier; files whose name starts with the identifier are skipped.\n'

  9. 	printf 'Configuration happens through a subdir/.dir-to-ia.config file, a Bash script that gets sourced to get the config values.\n'

  10. 	printf "Use '%q --example-config' to get an example configuration with explanations.\n" "$0"

  11. 	printf 'The upload log for each item gets written to the subdir/.dir-to-ia.log file.\n'

  12. 	printf 'It is assumed that no other changes are made to the items while dir-to-ia is running.\n'

  13. 	printf 'Multiple instances can safely run in the same directory to upload multiple items in parallel. Uploads within an item are always sequential.\n'

  14. 	printf '\n'

  15. 	printf 'This program is dedicated to fireonlive.\n'

  16. 	exit 1

  17. fi

  18. 

  19. # Default config values

  20. sha256=no

  21. rm=no

  22. rmwait=yes

  23. sha1check=no

  24. clobber=no

  25. derive=no

  26. autosizehint=no

  27. iauploadstreamopts=()

  28. 

  29. if [[ "$1" == '--example-config' ]]; then

  30. 	cat <<-EOF

  31. 		# Calculate SHA-256 hash of each file after uploading

  32. 		sha256=${sha256}

  33. 

  34. 		# Remove local file after successful upload; when disabled, only files without a filename match in the IA item get uploaded.

  35. 		rm=${rm}

  36. 

  37. 		# Delay removal until IA processed the upload

  38. 		rmwait=${rmwait}

  39. 

  40. 		# Run an SHA-1 check against the local file after the upload using the hash calculated by IA; if rm and sha1check are enabled, rmwait must also be enabled.

  41. 		sha1check=${sha1check}

  42. 

  43. 		# Clobber existing files in IA item (no = existing copy is moved to history/files/ by IA); only meaningful with rm=yes.

  44. 		clobber=${clobber}

  45. 

  46. 		# Queue derive after upload (applied after every file!)

  47. 		derive=${derive}

  48. 

  49. 		# Automatic size hint for item creation using the current directory size. When more data will be added, do not use this and set a value in iauploadstreamopts instead if a size hint is desired.

  50. 		autosizehint=${autosizehint}

  51. 

  52. 		# Custom options for ia-upload-stream, as an array of args; this can be used to choose the part size and concurrency, for example, using (--part-size 1G --concurrency 4).

  53. 		iauploadstreamopts=()

  54. 

  55. 		# Item metadata (array with 'key:value' elements); the only mandatory variable with no default

  56. 		metadata=('collection:opensource' "date:$(printf '%(%Y-%m-%d)T')")

  57. 	EOF

  58. 	exit

  59. fi

  60. 

  61. function tsprintf {

  62. 	# Bash 5.0+, ms is empty below that, will break in the year 2286

  63. 	local t="${EPOCHREALTIME:--1}" ts ms

  64. 	ms="${t:10:4}"

  65. 	printf -v ts '%(%Y-%m-%d %H:%M:%S)T%s' "${t%%.*}" "${ms}";

  66. 	if [[ $# -eq 1 ]]; then

  67. 		printf "%s %s\n" "${ts}" "$@";

  68. 	else

  69. 		# First argument is the format, rest are arguments to printf; pass through `while read` loop to have a better chance of line buffering.

  70. 		{

  71. 			printf "%s " "${ts}";

  72. 			printf "$1" "${@:2}";

  73. 			printf '\n';

  74. 		} | while IFS= read -r line; do printf '%s\n' "${line}"; done

  75. 	fi

  76. }

  77. 

  78. function maybets {

  79. 	# Add a timestamp to each line that doesn't start with one.

  80. 	local line

  81. 	while IFS= read -r line; do

  82. 		if [[ "${line}" != [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\ [0-9][0-9]:[0-9][0-9]:[0-9][0-9]* ]]; then

  83. 			tsprintf "${line}"

  84. 		else

  85. 			printf '%s\n' "${line}"

  86. 		fi

  87. 	done

  88. }

  89. 

  90. if [[ "${DEBUG}" ]]; then

  91. 	function dbgprintf { tsprintf "[DEBUG] $1" "${@:2}" >&2; }

  92. else

  93. 	function dbgprintf { :; }

  94. fi

  95. 

  96. scriptpath="$(cd "$(dirname "$0")"; pwd -P)"

  97. for cmd in ia-upload-stream ia-tasks; do

  98. 	if [[ ! -x "${scriptpath}/${cmd}" ]]; then

  99. 		printf 'Error: %q not found, make sure it is in the same directory as %q\n' "${cmd}" "$0" >&2

  100. 		exit 1

  101. 	fi

  102. done

  103. 

  104. while :; do

  105. 	# Prefilter the subdirs for ones that have actual files in them. This avoids cycling through all dirs when rm=yes.

  106. 	# Include dirs with .dirty file so the warning about those is always emitted.

  107. 	readarray -d $'\0' -t candidateDirs < <(find -type f \( \! -name '.dir-to-ia.*' -o -name .dir-to-ia.dirty \) -print0 | sed -z 's,^\./,,; s,/.*$,,' | uniq -z)

  108. 	for dir in "${candidateDirs[@]}"; do

  109. 		dbgprintf 'Considering directory %q' "${dir}"

  110. 		identifier="${dir}"

  111. 		if [[ ! "${dir}" =~ ^[a-zA-Z0-9] ]]; then

  112. 			continue

  113. 		fi

  114. 		if [[ ! -e "${dir}/.dir-to-ia.config" ]]; then

  115. 			continue

  116. 		fi

  117. 		if [[ -e "${dir}/.dir-to-ia.dirty" ]]; then

  118. 			tsprintf "Warning: %q is dirty, skipping" "${dir}" >&2

  119. 			continue

  120. 		fi

  121. 

  122. 		# Everything from here on is executed in a subshell so that the config sourcing can't affect other items.

  123. 		# It would be possible to do that without a subshell, but this is easier.

  124. 		{

  125. 			flock --exclusive --nonblock 3 || continue

  126. 			(

  127. 				dbgprintf 'Processing %q' "${dir}"

  128. 

  129. 				# Source and check configuration

  130. 				. "${dir}/.dir-to-ia.config" || { tsprintf 'Sourcing %q failed' "${dir}/.dir-to-ia.config" >&2; exit 1; }

  131. 				configbroken=

  132. 				for v in sha256 rm rmwait sha1check clobber derive; do

  133. 					if [[ "${!v}" != 'yes' && "${!v}" != 'no' ]]; then

  134. 						tsprintf "Error in %q: %s must be 'yes' or 'no'" "${dir}/.dir-to-ia.config" "${v}" >&2

  135. 						configbroken=yes

  136. 					fi

  137. 				done

  138. 				if read -r _ attrs _ < <(declare -p iauploadstreamopts 2>/dev/null); [[ "${attrs}" != *a* ]]; then

  139. 					tsprintf 'Error in %q: iauploadstreamopts must be an array' "${dir}/.dir-to-ia.config" >&2

  140. 					configbroken=yes

  141. 				fi

  142. 				if read -r _ attrs _ < <(declare -p metadata 2>/dev/null); [[ "${attrs}" != *a* ]]; then

  143. 					tsprintf 'Error in %q: metadata missing or not an array' "${dir}/.dir-to-ia.config" >&2

  144. 					configbroken=yes

  145. 				else

  146. 					for f in "${metadata[@]}"; do

  147. 						if [[ "${f}" != *:* ]]; then

  148. 							tsprintf 'Error in %q: metadata field missing colon: %s' "${dir}/.dir-to-ia.config" "${f}" >&2

  149. 							configbroken=yes

  150. 						fi

  151. 					done

  152. 				fi

  153. 				if [[ "${configbroken}" ]]; then

  154. 					exit 1

  155. 				fi

  156. 				if [[ "${rm}" == 'no' && "${clobber}" == 'yes' ]]; then

  157. 					tsprintf 'Error in %q: rm=no and clobber=yes is not permitted' "${dir}/.dir-to-ia.config" >&2

  158. 					exit 1

  159. 				fi

  160. 				if [[ "${rm}" == 'yes' && "${sha1check}" == 'yes' && "${rmwait}" == 'no' ]]; then

  161. 					tsprintf 'Error in %q: rm=yes + sha1check=yes is incompatible with rmwait=no' "${dir}/.dir-to-ia.config" >&2

  162. 					exit 1

  163. 				fi

  164. 				dbgprintf 'Configuration:'

  165. 				dbgprintf "  sha256=${sha256}"

  166. 				dbgprintf "  rm=${rm}"

  167. 				dbgprintf "  rmwait=${rmwait}"

  168. 				dbgprintf "  sha1check=${sha1check}"

  169. 				dbgprintf "  clobber=${clobber}"

  170. 				dbgprintf "  derive=${derive}"

  171. 				dbgprintf "  autosizehint=${autosizehint}"

  172. 				dbgprintf "  iauploadstreamopts=(${iauploadstreamopts[0]+"$(printf %q "${iauploadstreamopts[0]}")"}$(if [[ ${#iauploadstreamopts[@]} -gt 1 ]]; then printf ' %q' "${iauploadstreamopts[@]:1}"; fi))"

  173. 				dbgprintf "  metadata=($(printf %q "${metadata[0]}")$(if [[ ${#metadata[@]} -gt 1 ]]; then printf ' %q' "${metadata[@]:1}"; fi))"

  174. 

  175. 				dbgprintf 'Version: commit %q' "$(git -C "${scriptpath}" log --max-count=1 --format=%H)"

  176. 

  177. 				# Check for previous batch that might require waiting for IA and/or deletion

  178. 				# Exits 0 for likely IA-side errors so it gets retried on the next pass.

  179. 				if [[ -e "${dir}/.dir-to-ia.batch-files" ]]; then

  180. 					if [[ ( "${rm}" == 'yes' && "${rmwait}" == 'yes' ) || "${sha1check}" == 'yes' ]]; then

  181. 						tsprintf 'Checking whether IA is done with upload processing...' >&2

  182. 						summary="$("${scriptpath}/ia-tasks" --summary --jsonl "${identifier}")"

  183. 						st="$?"

  184. 						if ((st)); then

  185. 							tsprintf "Warning: task processing wait: ia-tasks exited with status ${st}" >&2

  186. 							exit 0

  187. 						fi

  188. 						tasksAndErrors="$(python3 -c 'import json, sys; o = json.load(sys.stdin); print(sum(o.values()), o["error"])' <<<"${summary}")"

  189. 						st="$?"

  190. 						if ((st)); then

  191. 							tsprintf "Warning: task processing wait: python3 for ia-tasks output %q exited with status %d" "${summary}" "${st}" >&2

  192. 							exit 0

  193. 						fi

  194. 						if [[ "${tasksAndErrors}" != '0 0' ]]; then

  195. 							tsprintf "Task processing wait: IA tasks not finished (%d tasks remaining, %d errors)" "${tasksAndErrors% *}" "${tasksAndErrors#* }" >&2

  196. 							exit 0

  197. 						fi

  198. 						tsprintf 'IA upload processing finished' >&2

  199. 					fi

  200. 

  201. 					readarray -d $'\0' -t uploaded <"${dir}/.dir-to-ia.batch-files"

  202. 					dbgprintf "Local files:$(printf " %q" "${uploaded[@]}")"

  203. 

  204. 					if [[ "${sha1check}" == 'yes' ]]; then

  205. 						tsprintf 'Checking SHA-1 hashes...' >&2

  206. 						if [[ ! -e "${dir}/.dir-to-ia.batch-sha1sums" ]]; then

  207. 							tsprintf 'Error: sha1check enabled but no sha1sums file for the previous batch of uploads (did you enable sha1check while a batch was pending?)' >&2

  208. 							exit 1

  209. 						fi

  210. 						readarray -t localhashes <"${dir}/.dir-to-ia.batch-sha1sums"

  211. 						dbgprintf "Local hashes:$(printf " %q" "${localhashes[@]}")"

  212. 						if ((${#localhashes[@]} != ${#uploaded[@]})); then

  213. 							tsprintf 'Error: %q and %q do not have the same number of entries (%d vs %d)' "${dir}/.dir-to-ia.batch-files" "${dir}/.dir-to-ia.batch-sha1sums" "${#uploaded[@]}" "${#localhashes[@]}" >&2

  214. 							exit 1

  215. 						fi

  216. 						dbgprintf 'Retrieving SHA-1 hashes from IA...'

  217. 						# Only two error scenarios: retrieval fails (curl exit != 0), or the file list doesn't contain all files in the array (Python exit 33)

  218. 						# To not have to handle weird characters in filenames, don't try to produce a file with the expected format for `sha1sum -c`.

  219. 						shopt -s lastpipe

  220. 						curl --silent --location --max-time 10 --fail --user-agent 'little-things +https://gitea.arpa.li/JustAnotherArchivist/little-things' "https://archive.org/metadata/${identifier}" \

  221. 						| python3 -c 'import json, sys; o = json.load(sys.stdin); files = set(sys.argv[1:]); {(print(f["sha1"], end = "\0"), print(f["name"], end = "\0"), files.remove(f["name"])) for f in o.get("files", []) if f["name"] in files and "sha1" in f}; sys.exit(33) if files else ...' "${uploaded[@]}" \

  222. 						| readarray -d $'\0' -t iahashes

  223. 						sts=("${PIPESTATUS[@]}")

  224. 						dbgprintf "IA hashes:$(printf " %q" "${iahashes[@]}")"

  225. 						if ((sts[0])); then

  226. 							tsprintf "IA hashes curl exited with status ${sts[0]}" >&2

  227. 							exit 0

  228. 						fi

  229. 						if ((sts[1] == 33)); then

  230. 							tsprintf 'IA hashes metadata does not contain all files uploaded in this batch' >&2

  231. 							exit 1

  232. 						elif ((sts[1])); then

  233. 							tsprintf "IA hashes python3 exited with status ${sts[1]}" >&2

  234. 							exit 0

  235. 						fi

  236. 						if ((${#iahashes[@]} % 2 != 0)); then

  237. 							tsprintf 'Got an odd number of hashes from IA' >&2

  238. 							exit 1

  239. 						fi

  240. 						if ((${#iahashes[@]} != 2 * ${#localhashes[@]})); then

  241. 							tsprintf "Number of hashes from IA (%d) doesn't match number of local hashes (%d)" "$((${#iahashes[@]} / 2))" "${#localhashes[@]}" >&2

  242. 							exit 1

  243. 						fi

  244. 

  245. 						# Local filenames in uploaded, corresponding local hashes + space space dash in localhashes, and iahashes is (hash filename [hash filename...]).

  246. 						dbgprintf 'Comparing...'

  247. 						hashfail=

  248. 						for ((i = 0; i < ${#uploaded[@]}; ++i)); do

  249. 							fn="${uploaded[i]}"

  250. 							dbgprintf 'Checking hash for %q' "${dir}/${fn}"

  251. 							localhash="${localhashes[i]%  -}"

  252. 							dbgprintf '  local: %s' "${localhash}"

  253. 							iahash=

  254. 							for ((j = 0; j < "${#iahashes[@]}"; j += 2)); do

  255. 								if [[ "${iahashes[j + 1]}" == "${fn}" ]]; then

  256. 									iahash="${iahashes[j]}"

  257. 									break

  258. 								fi

  259. 							done

  260. 							if [[ -z "${iahash}" ]]; then

  261. 								# This should be impossible due to the checks above.

  262. 								tsprintf 'Error: could not find IA hash for %q' "${fn}" >&2

  263. 								exit 1

  264. 							fi

  265. 							dbgprintf '  IA: %s' "${iahash}"

  266. 							if [[ "${iahash}" != "${localhash}" ]]; then

  267. 								tsprintf 'SHA-1 mismatch for %q: local %s vs IA %s' "${dir}/${fn}" "${localhash}" "${iahash}" >&2

  268. 								hashfail=yes

  269. 							else

  270. 								tsprintf 'OK: %q' "${dir}/${fn}" >&2

  271. 							fi

  272. 						done

  273. 						if [[ "${hashfail}" ]]; then

  274. 							exit 1

  275. 						fi

  276. 					fi

  277. 

  278. 					if [[ "${rm}" == 'yes' ]]; then

  279. 						tsprintf 'Removing:' >&2

  280. 						for fn in "${uploaded[@]}"; do

  281. 							tsprintf '  %q' "${dir}/${fn}" >&2

  282. 							rm "${dir}/${fn}" || { tsprintf 'rm for %q exited with status %d' "${dir}/${fn}" "$?" >&2; exit 1; }

  283. 						done

  284. 					fi

  285. 

  286. 					rm "${dir}/.dir-to-ia.batch-files" || { tsprintf "rm for %q batch files or sha1sums exited with status %d" "${dir}" "$?" >&2; exit 1; }

  287. 					if [[ -e "${dir}/.dir-to-ia.batch-sha1sums" ]]; then

  288. 						rm "${dir}/.dir-to-ia.batch-sha1sums" || { tsprintf "rm for %q batch files or sha1sums exited with status %d" "${dir}" "$?" >&2; exit 1; }

  289. 					fi

  290. 				fi

  291. 

  292. 				# Loop over local files and upload them

  293. 				# The 'quota' controls how many files get uploaded at once.

  294. 				# It gets decremented for each uploaded file, and when it reaches zero, the item's task queue is checked to ensure it isn't building up.

  295. 				gotFileList=

  296. 				nQuota=50

  297. 				while IFS= read -r -d $'\0' fn; do

  298. 					dbgprintf 'Considering file %q' "${fn}"

  299. 					if [[ "${fn}" == .dir-to-ia.* ]]; then

  300. 						dbgprintf '%q is a dir-to-ia file, skipping' "${fn}"

  301. 						continue

  302. 					fi

  303. 					if [[ ! -f "${dir}/${fn}" ]]; then

  304. 						# Should never happen since the `find` command already uses `-type f`

  305. 						dbgprintf '%q is not a regular file, skipping' "${fn}"

  306. 						continue

  307. 					fi

  308. 					if [[ "${fn}" == "${identifier}"* ]]; then

  309. 						dbgprintf '%q starts with the identifier, skipping' "${fn}"

  310. 						continue

  311. 					fi

  312. 					if [[ "${rm}" == 'no' ]]; then

  313. 						if [[ ! "${gotFileList}" ]]; then

  314. 							# This can only proceed if there are no pending tasks; if there are, break out of the loop and revisit later.

  315. 							dbgprintf 'Checking whether IA has pending tasks on the item as the item file list/metadata (needed for rm=no) is unreliable otherwise...'

  316. 							summary="$("${scriptpath}/ia-tasks" --summary --jsonl"${identifier}")"

  317. 							st="$?"

  318. 							if ((st)); then

  319. 								tsprintf "Warning: metadata reliability check: ia-tasks exited with status ${st}" >&2

  320. 								break

  321. 							fi

  322. 							tasksAndErrors="$(python3 -c 'import json, sys; o = json.load(sys.stdin); print(sum(o.values()), o["error"])' <<<"${summary}")"

  323. 							st="$?"

  324. 							if ((st)); then

  325. 								tsprintf "Warning: metadata reliability check: python3 for ia-tasks output %q exited with status %d" "${summary}" "${st}" >&2

  326. 								break

  327. 							fi

  328. 							if [[ "${tasksAndErrors}" != '0 0' ]]; then

  329. 								dbgprintf "Metadata reliability check: IA tasks not finished (%d tasks remaining, %d errors)" "${tasksAndErrors% *}" "${tasksAndErrors#* }" >&2

  330. 								break

  331. 							fi

  332. 							dbgprintf 'No pending tasks, metadata should be reliable' >&2

  333. 

  334. 							dbgprintf 'Retrieving existing files on IA...'

  335. 							# Scenarios: retrieval fails (curl exit != 0), item doesn't exist (= empty JSON, Python exit 31), no file list (e.g. darked, Python exit 32)

  336. 							shopt -s lastpipe

  337. 							curl --silent --location --max-time 10 --fail --user-agent 'little-things +https://gitea.arpa.li/JustAnotherArchivist/little-things' "https://archive.org/metadata/${identifier}" \

  338. 							| python3 -c 'import json, sys; o = json.load(sys.stdin); sys.exit(31) if o == {} else ...; sys.exit(32) if "files" not in o else ...; {print(f["name"], end = "\0") for f in o["files"] if not f["name"].startswith(sys.argv[1])}' "${identifier}" \

  339. 							| readarray -d $'\0' -t iafiles

  340. 							sts=("${PIPESTATUS[@]}")

  341. 							dbgprintf "Existing files on IA:$(printf " %q" "${iafiles[@]}")"

  342. 							if ((sts[0])); then

  343. 								tsprintf "IA metadata curl exited with status ${sts[0]}" >&2

  344. 							fi

  345. 							if ((sts[1] == 31)); then

  346. 								# Item doesn't exist, but that's fine. Reset the status so it doesn't trigger the exit below.

  347. 								sts[1]=0

  348. 							elif ((sts[1] == 32)); then

  349. 								tsprintf "IA returned no file list in metadata" >&2

  350. 							elif ((sts[1])); then

  351. 								tsprintf "IA metadata python3 exited with status ${sts[1]}" >&2

  352. 							fi

  353. 							if ((sts[0] || sts[1])); then

  354. 								break

  355. 							fi

  356. 							gotFileList=yes

  357. 						fi

  358. 

  359. 						found=

  360. 						for remoteFn in "${iafiles[@]}"; do

  361. 							if [[ "${fn}" == "${remoteFn}" ]]; then

  362. 								found=yes

  363. 								break

  364. 							fi

  365. 						done

  366. 						if [[ "${found}" ]]; then

  367. 							dbgprintf '%q found in IA item, skipping' "${fn}"

  368. 							continue

  369. 						fi

  370. 					fi

  371. 

  372. 					tsprintf 'Uploading %q to %q...' "${fn}" "${identifier}" >&2

  373. 					cmd=("${scriptpath}/ia-upload-stream")

  374. 					if [[ "${clobber}" == 'yes' ]]; then cmd+=('--clobber'); fi

  375. 					if [[ "${derive}" == 'no' ]]; then cmd+=('--no-derive'); fi

  376. 					if [[ "${autosizehint}" == 'yes' ]]; then

  377. 						read -r size _ < <(du -s -k "${dir}")

  378. 						cmd+=('--size-hint' "${size}K")

  379. 					fi

  380. 					cmd+=(--input-file "${dir}/${fn}")

  381. 					cmd+=("${iauploadstreamopts[@]}")

  382. 					cmd+=("${identifier}")

  383. 					cmd+=("${fn}")

  384. 					cmd+=("${metadata[@]}")

  385. 

  386. 					dbgprintf "Upload command:$(printf " %q" "${cmd[@]}")"

  387. 					"${cmd[@]}" || { tsprintf "ia-upload-stream exited with status $?" >&2; exit 1; }

  388. 

  389. 					if [[ "${sha256}" == 'yes' ]]; then

  390. 						tsprintf 'Calculating SHA-256...' >&2

  391. 						(cd "${dir}" && sha256sum "${fn}" > >(tee -a '.dir-to-ia.sha256sum')) || { tsprintf 'sha256sum for %q exited with status %d' "${fn}" "$?" >&2; exit 1; }

  392. 					fi

  393. 					if [[ "${sha1check}" == 'yes' ]]; then

  394. 						tsprintf 'Calculating SHA-1...' >&2

  395. 						sha1sum <"${dir}/${fn}" > >(tee -a "${dir}/.dir-to-ia.batch-sha1sums") || { tsprintf 'sha1sum for %q exited with status %d' "${fn}" "$?" >&2; exit 1; }

  396. 					fi

  397. 

  398. 					if [[ "${rm}" == 'yes' || "${sha1check}" == 'yes' ]]; then

  399. 						printf '%s\0' "${fn}" >>"${dir}/.dir-to-ia.batch-files"

  400. 					fi

  401. 

  402. 					((--nQuota))

  403. 					if ((nQuota == 0)); then

  404. 						dbgprintf "Checking IA's task queue to determine how many more files we can upload in this iteration..."

  405. 						summary="$("${scriptpath}/ia-tasks" --summary --jsonl "${identifier}")"

  406. 						st="$?"

  407. 						if ((st)); then

  408. 							tsprintf "Warning: task queue check: ia-tasks exited with status ${st}" >&2

  409. 							exit 0

  410. 						fi

  411. 						tasksAndErrors="$(python3 -c 'import json, sys; o = json.load(sys.stdin); print(sum(o.values()), o["error"])' <<<"${summary}")"

  412. 						st="$?"

  413. 						if ((st)); then

  414. 							tsprintf "Warning: task queue check: python3 for ia-tasks output %q exited with status %d" "${summary}" "${st}" >&2

  415. 							exit 0

  416. 						fi

  417. 						tasks="${tasksAndErrors% *}"

  418. 						errors="${tasksAndErrors#* }"

  419. 						((nQuota = 50 - tasks))

  420. 						if ((nQuota < 0 || errors > 0)); then

  421. 							nQuota=0

  422. 						fi

  423. 						dbgprintf "IA item has %d tasks remaining and %d errors, new quota: %d" "${tasks}" "${errors}" "${nQuota}"

  424. 						if ((nQuota == 0)); then

  425. 							break

  426. 						fi

  427. 					fi

  428. 				done < <(cd "${dir}" && find . -type f -print0 | sed -z 's,^\./,,')

  429. 

  430. 				# Waiting for IA and/or deletions are handled in the next iteration

  431. 			)

  432. 			st="$?"

  433. 			if ((st)); then

  434. 				tsprintf 'Subshell exited with status %d, marking as dirty' "${st}" >&2

  435. 				printf '%s\n' \

  436. 					'An error occurred on the last upload, so this file marks this directory as dirty to not cause worse problems.' \

  437. 					'Delete this file once the underlying issue is resolved to make dir-to-ia process this directory again.' \

  438. 					>"${dir}/.dir-to-ia.dirty"

  439. 			fi

  440. 			dbgprintf 'Done with %q' "${dir}"

  441. 		} 3>"${dir}/.dir-to-ia.lock" &> >(stdbuf -oL tr '\r' '\n' | maybets | tee -a "${dir}/.dir-to-ia.log" >&2)

  442. 	done

  443. 	sleep 60

  444. done