From 10e5716c2011976b588e16025cdc6822912ac446 Mon Sep 17 00:00:00 2001 From: Ivan Kozik Date: Thu, 11 Apr 2024 23:51:30 +0000 Subject: [PATCH] WIP sftp-into-eye --- sftp-into-eye | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sftp-into-eye b/sftp-into-eye index 018aec1..9e6aa13 100755 --- a/sftp-into-eye +++ b/sftp-into-eye @@ -1,7 +1,19 @@ -#!/usr/bin/env bash +#!/usr/bin/env zsh set -eu -o pipefail +# Enable job control for wait-until-jobs-below +set -m + +# Wait until the number of shell jobs drops below a limit. +# This can be used as an alternative to parallel using +# for i in ...; do something $i &; wait-until-jobs-below 8; done +wait-until-jobs-below() { + while test $(jobs | wc -l) -ge "$1"; do + sleep 0.1 + done +} + # e.g. ~/stash/YouTube/whitehouse root=$1 @@ -18,8 +30,8 @@ es x find -t f -- "$root" | take-wanted-ids ~/dead-video-ids | rg '\.(mp4|webm|f es x get "$i" && \ rclone --inplace --size-only --progress copyto "$i" "jeff:$eye_path" && \ curl --fail-with-body -u "ya: ." -X POST -d "{\"file_id\": $file_id, \"location\": \"the-eye\", \"pathname\": \"$eye_path\"}" https://ya.borg.xyz/cgi-bin/new-namedfiles && \ - curl -X POST -d "{\"the_eye_any_bytes_saved\": $size, \"the_eye_video_bytes_saved\": $size, \"the_eye_any_files_saved\": 1, \"the_eye_video_files_saved\": 1}" "http://127.0.0.1:31416/metrics" && \ - ) || true & + curl -X POST -d "{\"the_eye_any_bytes_saved\": $size, \"the_eye_video_bytes_saved\": $size, \"the_eye_any_files_saved\": 1, \"the_eye_video_files_saved\": 1}" "http://127.0.0.1:31416/metrics" || true + ) & wait-until-jobs-below 9 done