download-scripts/sftp-into-eye

#!/usr/bin/env zsh

set -eu -o pipefail

# Enable job control for wait-until-jobs-below
set -m

# Wait until the number of shell jobs drops below a limit.
# This can be used as an alternative to parallel using
# for i in ...; do something $i &; wait-until-jobs-below 8; done
wait-until-jobs-below() {
	while test $(jobs | wc -l) -ge "$1"; do
		sleep 0.1
	done
}

# e.g. ~/stash/YouTube/whitehouse
root=$1

es x find -t f -- "$root" | take-wanted-ids ~/dead-video-ids | rg '\.(mp4|webm|flv|video)$' | while read i; do
	echo $i
	eye_path=$(path-to-eye-path "$i" | sed -r "s,$HOME/stash/,,g")
	file_id=$(es x info -- "$i" | jq .id)
	size=$(es x info -- "$i" | jq .size)

	# If already in namedfiles i.e. the-eye, skip
	es x info -- "$i" | rg -q -F '"type": "namedfiles"' && continue || true

	(
		es x get -s -- "$i" && \
		rclone --inplace --size-only --progress copyto "$i" "jeff:$eye_path" && \
		curl --fail-with-body -u "ya:   ." -X POST -d "{\"file_id\": $file_id, \"location\": \"the-eye\", \"pathname\": \"$eye_path\"}" https://ya.borg.xyz/cgi-bin/new-namedfiles && \
		curl -X POST -d "{\"the_eye_any_bytes_saved\": $size, \"the_eye_video_bytes_saved\": $size, \"the_eye_any_files_saved\": 1, \"the_eye_video_files_saved\": 1}" "http://127.0.0.1:31416/metrics" || true;
		rm -f -- "$i"
	) &

	wait-until-jobs-below 16
done