From 851e7b7241c3c208f974f03bdaa122479a538244 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Thu, 24 Sep 2020 00:41:06 -0400
Subject: [PATCH] Remove delays in starting all threads, delay in retrieving
 new items

---
 worker.py | 44 +++++++++++++-------------------------------
 1 file changed, 13 insertions(+), 31 deletions(-)

diff --git a/worker.py b/worker.py
index 88747fe..d674e21 100644
--- a/worker.py
+++ b/worker.py
@@ -17,8 +17,6 @@ from queue import Queue
 
 from gc import collect
 
-from datetime import timedelta, datetime
-
 from discovery import getmetadata
 from export import subprrun
 
@@ -27,8 +25,6 @@ jobs = Queue()
 
 langcnt = {}
 
-lasttask = datetime.min
-
 try:
     mkdir("out")
 except:
@@ -221,41 +217,27 @@ def threadrunner(jobs: Queue):
             jobs.task_done()
         else:
             # get a new task from tracker
-            if datetime.now() - lasttask > timedelta(seconds=15): #only retrieve a task every 15 seconds to allow queue to build up
-                collect() #cleanup
-                desit = tracker.request_item_from_tracker()
-                print("New task:", desit)
-                if desit:
-                    if desit.split(":", 1)[0] == "video":
-                        lasttask = datetime.now()
-                        jobs.put(("discovery", desit.split(":", 1)[1], None))
-                    elif desit.split(":", 1)[0] == "channel":
-                        lasttask = datetime.now()
-                        jobs.put(("channel", None, desit.split(":", 1)[1]))
-                    elif desit.split(":", 1)[0] == "playlist":
-                        lasttask = datetime.now()
-                        jobs.put(("playlist", None, desit.split(":", 1)[1]))
-                    else:
-                        print("Ignoring item for now", desit)
+            collect() #cleanup
+            desit = tracker.request_item_from_tracker()
+            print("New task:", desit)
+            if desit:
+                if desit.split(":", 1)[0] == "video":
+                    jobs.put(("discovery", desit.split(":", 1)[1], None))
+                elif desit.split(":", 1)[0] == "channel":
+                    jobs.put(("channel", None, desit.split(":", 1)[1]))
+                elif desit.split(":", 1)[0] == "playlist":
+                    jobs.put(("playlist", None, desit.split(":", 1)[1]))
                 else:
                     print("Ignoring item for now", desit)
             else:
-                sleep(1)
+                print("Ignoring item for now", desit)
     
 
 threads = []
 
-#start with 1 thread, give it a 5 second head start
-runthread = Thread(target=threadrunner, args=(jobs,))
-runthread.start()
-threads.append(runthread)
-del runthread
-
-sleep(5)
-
-THREADCNT = 49
+THREADCNT = 50
 if HEROKU:
-    THREADCNT = 19
+    THREADCNT = 20
 #now create the rest of the threads
 for i in range(THREADCNT):
     runthread = Thread(target=threadrunner, args=(jobs,))