|
- #!/bin/bash
- # Usage: archivebot-high-resources
- # Prints the CPU, RAM, and disk usage (log + DB) of all running jobs in a table with particularly bad values highlighted in red
- # If stdout is not a TTY, the output is not tabularised or coloured.
- # Output order is arbitrary.
-
- function get_utime_rss_du {
- # Non-empty argument 1 disables du
- while read -r pid jobid path rss; do
- # utime in clock ticks
- utime="$(cut -d ' ' -f 14 <"/proc/${pid}/stat")"
- if [[ "$1" ]]; then
- #TODO This is kind of slow; is there a faster way?
- du="$(($(du -k -s "${path}"/wpull.* | cut -d $'\t' -f 1 | tr '\n' '+'; echo 0)))"
- else
- du=-1
- fi
- printf '%s %s %s %s %s %s\n' "$1" "${pid}" "${jobid}" "${utime}" "${rss}" "${du}"
- done < <(ps -a -o pid,rss,cmd | grep '[w]pull' | sed 's,^\s*\([0-9][0-9]*\)\s\s*\([0-9][0-9]*\)\s\s*.*\s\([^\s]*/\([0-9a-z][0-9a-z]*\)\)/[w]pull\.log\s.*$,\1 \4 \3 \2,')
- }
-
- interval=5
- clk_tck="$(getconf CLK_TCK)"
- [ ! -t 1 ]; stdout_tty=$? # 0 = is not tty, 1 = is tty
-
- {
- get_utime_rss_du before no-du
- sleep "${interval}"
- get_utime_rss_du after
- } | python3 -c \
- "$(cat <<'EOF'
-
- import sys
-
- CLK_TCK = int(sys.argv[1])
- INTERVAL = int(sys.argv[2])
- STDOUT_TTY = bool(int(sys.argv[3]))
-
- RED = '\x1b[0;31m' if STDOUT_TTY else ''
- RESET = '\x1b[0m' if STDOUT_TTY else ''
-
- stats = {"before": {}, "after": {}}
- for t, pid, jobid, utime, rss, du in map(str.split, map(str.strip, sys.stdin)):
- stats[t][f"{pid}_{jobid}"] = (int(pid), jobid, int(utime), int(rss), int(du))
-
- print('PID', 'JOBID', 'CPU[%]', 'RSS[KiB]', 'DISK[KiB]')
- for key in set(stats['before'].keys()) & set(stats['after'].keys()):
- pid, jobid, utime0, _, _ = stats['before'][key]
- _, _, utime1, rss, du = stats['after'][key]
- cpup = (utime1 - utime0) / CLK_TCK / INTERVAL * 100
- print('{} {} {}{:.1f}{} {}{}{} {}{}{}'.format(pid, jobid, RED if cpup >= 75 else '', cpup, RESET, RED if rss >= 200_000 else '', rss, RESET, RED if du >= 10_000_000 else '', du, RESET))
-
- EOF
- )" \
- "${clk_tck}" \
- "${interval}" \
- "${stdout_tty}" \
- | if [[ "${stdout_tty}" == 1 ]]; then column -t; else cat; fi
|