diff --git a/codearchiver-bot b/codearchiver-bot index a05cd67..cbe08d2 100755 --- a/codearchiver-bot +++ b/codearchiver-bot @@ -140,9 +140,6 @@ function respond { continue fi - # Marker for finding new files - touch '.filemarker' - # Find nonexistent filename for log file (*not* concurrency-safe!) logbasename="$(date +%Y%m%dT%H%M%SZ)_${singlejobid}" if [[ -e "${logbasename}_codearchiver.log" ]]; then @@ -154,10 +151,13 @@ function respond { logbasename="${logbasename}_coll${i}" fi logname="${logbasename}_codearchiver.log" + artefactsname="${logbasename}_codearchiver_artefacts.txt" # Run codearchiver, duplicating WARNINGs and higher in the bot output - log "Running ${url}, logging into ${logname}" - codearchiver --verbose "${url}" 2> >(tee "${logname}" | grep -Fv -e ' INFO ' | log_loop "From codearchiver ${singlejobid}: ") + log "Running ${url} (${singlejobid}), logging into ${logname}" + codearchiver --verbose --write-artefacts-fd-3 "${url}" \ + 2> >(tee "${logname}" | grep -Fv -e ' INFO ' | log_loop "From codearchiver ${singlejobid}: ") \ + 3> >(tee "${artefactsname}" | log_loop "New artefacts from codearchiver ${singlejob}: ") status="$?" log "codearchiver ${url} finished with status code ${status}" statuscodes+=("${status}") @@ -172,29 +172,23 @@ function respond { logname="${logname}.zst" fi - # Collect new files, write artefacts file, move everything but the log file to ./failed/ if codearchiver exited non-zero - readarray -d $'\0' -t tmpnewfiles < <(find -maxdepth 1 -type f -newer '.filemarker' -print0 2> >(log_loop 'find new files err: ') | sed -z 's,^\./,,') - readarray -d $'\0' -t artefacts < <(for file in "${tmpnewfiles[@]}"; do if [[ "${file}" != "${logname}" ]]; then printf '%s\0' "${file}"; fi; done) - artefactsname="${logbasename}_codearchiver_artefacts.txt" - for file in "${artefacts[@]}"; do - printf '%s\n' "${file}" - done >"${artefactsname}" + # Move everything but the log file to ./failed/ if codearchiver exited non-zero + readarray -t artefacts <"${artefactsname}" if [[ "${status}" -ne 0 ]]; then msg="$(printf 'Moving artefact files'; printf ' %q' "${artefacts[@]}" "${artefactsname}"; printf ' from non-zero exit for job %s to ./failed/\n' "${singlejobid}";)" log "${msg}" mkdir --parents ./failed/ mv --verbose -- "${artefacts[@]}" "${artefactsname}" ./failed/ 2> >(log_loop 'mv err: ') | log_loop 'mv out: ' - newfiles+=("${logname}") - newfilejobindices+=("${i}") else - for file in "${tmpnewfiles[@]}"; do - log "New file from job ${singlejobid}: ${file}" + for file in "${artefacts[@]}"; do newfiles+=("${file}") newfilejobindices+=("${i}") done newfiles+=("${artefactsname}") newfilejobindices+=("${i}") fi + newfiles+=("${logname}") + newfilejobindices+=("${i}") # For individual jobs, tell the user about warnings and success/failure if [[ "${singlejobid}" != *_* ]]; then