The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

27 lines
1.1 KiB

  1. #!/usr/bin/env python3
  2. import os
  3. import redis
  4. import sqlite3
  5. assert os.path.exists('wpull.db'), 'no wpull.db file in this directory'
  6. jobid = os.getcwd().rsplit('/', 1)[-1]
  7. r = redis.StrictRedis.from_url(os.environ.get('REDIS_URL', 'redis://127.0.0.1:16379/0'), decode_responses = True)
  8. curDownloaded, curQueued = r.hmget(jobid, 'items_downloaded', 'items_queued')
  9. assert curDownloaded is not None and curQueued is not None, f'could not fetch downloaded and/or queued count for {jobid}'
  10. print(f'Current control node values: {curDownloaded}/{curQueued} (net {int(curQueued) - int(curDownloaded)})')
  11. db = sqlite3.connect('wpull.db')
  12. cur = db.cursor()
  13. res = cur.execute('SELECT (SELECT MAX(id) FROM queued_urls) AS total, (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "todo") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "error") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "in_progress") AS queued')
  14. total, queued = res.fetchone()
  15. print(f'Setting new values: {total - queued}/{total} (net {queued})')
  16. print(r.hmset(jobid, {'items_downloaded': total - queued, 'items_queued': total}))
  17. print('Done')