From 9d3b4e98562e4aa2aabff88370929ac4e9986138 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 18:47:38 -0400
Subject: [PATCH 01/13] WIP microtasks

---
 worker.py | 144 +++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 93 insertions(+), 51 deletions(-)

diff --git a/worker.py b/worker.py
index a134c95..08b0915 100644
--- a/worker.py
+++ b/worker.py
@@ -20,47 +20,16 @@ from gc import collect
 from discovery import getmetadata
 from export import subprrun
 
-batchcontent = []
-actualitems = []
+#useful Queue example: https://stackoverflow.com/a/54658363
+jobs = Queue()
+
+langcnt = {}
+
 
 HEROKU = False
 if isfile("../Procfile"):
     HEROKU = True
 
-def batchfunc():
-    ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})
-    
-    if not HEROKU:
-        desqsize = 51
-    elif HEROKU:
-        desqsize = 251
-    
-    while jobs.qsize() < desqsize:
-        desit = tracker.request_item_from_tracker()
-        if desit:
-            if desit.split(":", 1)[0] == "video":
-                jobs.put(desit.split(":", 1)[1])
-            elif desit.split(":", 1)[0] == "channel":
-                y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
-                for itemyv in y["entries"]:
-                    tracker.add_item_to_tracker(tracker.ItemType.Video, itemyv["id"])
-            elif desit.split(":", 1)[0] == "playlist":
-                y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)
-                for itemyvp in y["entries"]:
-                    tracker.add_item_to_tracker(tracker.ItemType.Video, itemyvp["id"])
-            else:
-                print("Ignoring item for now", desit)
-        else:
-            print("Ignoring item for now", desit)
-        
-        batchcontent.append(desit.split(":", 1)[1])
-        actualitems.append(desit)
-
-def submitfunc(submitqueue):
-    while not submitqueue.empty():
-        itype, ival = submitqueue.get()
-        tracker.add_item_to_tracker(itype, ival)
-
 langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 
     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 
     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 
@@ -73,16 +42,6 @@ langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az
     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 
     'xh', 'yi', 'yo', 'zu']
 
-#useful Queue example: https://stackoverflow.com/a/54658363
-jobs = Queue()
-
-ccenabledl = []
-
-recvids  = set()
-recchans = set()
-recmixes = set()
-recplayl = set()
-
 #HSID, SSID, SID cookies required
 if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():
     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}
@@ -128,6 +87,93 @@ class GracefulKiller:
 
 gkiller = GracefulKiller()
 
+#minitasks
+def threadrunner(jobs: Queue):
+    global langcnt
+    ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})
+    while not gkiller.kill_now:
+        if not jobs.empty():
+            task, vid, args = jobs.get()
+
+            if task == "submitdiscovery":
+                tracker.add_item_to_tracker(args, vid)
+            elif task == "discovery":
+                pass
+            elif task == "subtitles":
+                pass
+            elif task == "channel":
+                y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
+                for itemyv in y["entries"]:
+                    jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))
+                jobs.put(("complete", None, "channel:"+args))
+            elif task == "playlist":
+                y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)
+                for itemyvp in y["entries"]:
+                    jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))
+                jobs.put(("complete", None, "playlist:"+args))
+            elif task == "complete":
+                size = 0
+                if ":" in args:
+                    if args.split(":", 1)[0] == "video":
+                        if isfile("directory/"+args.split(":", 1)[1]+".zip"):
+                            size = getsize("directory/"+args.split(":", 1)[1]+".zip")
+                tracker.mark_item_as_done(args, size)
+        else:
+            # get a new task from tracker
+            desit = tracker.request_item_from_tracker()
+            if desit:
+                if desit.split(":", 1)[0] == "video":
+                    jobs.put(("discovery", desit.split(":", 1)[1], None))
+                elif desit.split(":", 1)[0] == "channel":
+                    jobs.put(("channel", None, desit.split(":", 1)[1]))
+                elif desit.split(":", 1)[0] == "playlist":
+                    jobs.put(("playlist", None, desit.split(":", 1)[1]))
+                else:
+                    print("Ignoring item for now", desit)
+            else:
+                print("Ignoring item for now", desit)
+            
+            batchcontent.append(desit.split(":", 1)[1])
+            actualitems.append(desit)
+
+
+
+
+batchcontent = []
+actualitems = []
+
+
+def batchfunc():
+    
+    
+    if not HEROKU:
+        desqsize = 51
+    elif HEROKU:
+        desqsize = 251
+    
+    while jobs.qsize() < desqsize:
+
+
+def submitfunc(submitqueue):
+    while not submitqueue.empty():
+        itype, ival = submitqueue.get()
+        tracker.add_item_to_tracker(itype, ival)
+
+
+
+
+
+ccenabledl = []
+
+recvids  = set()
+recchans = set()
+recmixes = set()
+recplayl = set()
+
+
+
+
+
 def prrun():
     while not jobs.empty():
         global recvids
@@ -314,11 +360,7 @@ while not gkiller.kill_now:
 
     # Report the batch as complete
     for itemb in actualitems:
-        size = 0
-        if ":" in itemb:
-            if itemb.split(":", 1)[0] == "video":
-                if isfile("directory/"+itemb.split(":", 1)[1]+".zip"):
-                    size = getsize("directory/"+itemb.split(":", 1)[1]+".zip")
+
         tracker.mark_item_as_done(itemb, size)
 
     # clear the output directories

From fb4b423da25103a0f8fd7f40d779d989b4e3cb06 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 20:26:38 -0400
Subject: [PATCH 02/13] More WIP

---
 export.py | 374 +++++++++++++++++++++++++++---------------------------
 worker.py |  45 +++++--
 2 files changed, 216 insertions(+), 203 deletions(-)

diff --git a/export.py b/export.py
index 309f010..76053dd 100644
--- a/export.py
+++ b/export.py
@@ -79,206 +79,202 @@ class MyHTMLParser(HTMLParser):
         elif self.get_starttag_text() and self.get_starttag_text().startswith('<div id="original-video-title"'):
             self.inittitle += data
 
-def subprrun(jobs, mysession):
-    while not jobs.empty():
-        collect() #cleanup memory
-        langcode, vid, mode = jobs.get()
-        vid = vid.strip()
-        print(langcode, vid)
-
-        while True:
-            if mode == "default":
-                pparams = (
-                    ("v", vid),
-                    ("lang", langcode),
-                    ("action_mde_edit_form", 1),
-                    ("bl", "vmp"),
-                    ("ui", "hd"),
-                    ("tab", "captions"),
-                    ("o", "U")
-                )
-
-                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-            elif mode == "forceedit-metadata":
-                pparams = (
-                    ("v", vid),
-                    ("lang", langcode),
-                    ("action_mde_edit_form", 1),
-                    ('forceedit', 'metadata'),
-                    ('tab', 'metadata')
-                )
-
-                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-            elif mode == "forceedit-captions":
-                pparams = (
-                    ("v", vid),
-                    ("lang", langcode),
-                    ("action_mde_edit_form", 1),
-                    ("bl", "vmp"),
-                    ("ui", "hd"),
-                    ('forceedit', 'captions'),
-                    ("tab", "captions"),
-                    ("o", "U")
-                )
-
-                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-
-            if not "accounts.google.com" in page.url:
-                break
-            else:
-                print("[Retrying in 30 seconds] Please supply authentication cookie information in config.json or environment variables. See README.md for more information.")
-                sleep(30)
-
-        inttext = page.text
-
-        try:
-            initlang = page.text.split("'metadataLanguage': \"", 1)[1].split('"', 1)[0]
-        except:
-            initlang = ""
-
-        del page
+def subprrun(jobs, mysession, langcode, vid, mode):
+    collect() #cleanup memory
+    vid = vid.strip()
+    print(langcode, vid)
+
+    while True:
+        if mode == "default":
+            pparams = (
+                ("v", vid),
+                ("lang", langcode),
+                ("action_mde_edit_form", 1),
+                ("bl", "vmp"),
+                ("ui", "hd"),
+                ("tab", "captions"),
+                ("o", "U")
+            )
+
+            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+        elif mode == "forceedit-metadata":
+            pparams = (
+                ("v", vid),
+                ("lang", langcode),
+                ("action_mde_edit_form", 1),
+                ('forceedit', 'metadata'),
+                ('tab', 'metadata')
+            )
+
+            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+        elif mode == "forceedit-captions":
+            pparams = (
+                ("v", vid),
+                ("lang", langcode),
+                ("action_mde_edit_form", 1),
+                ("bl", "vmp"),
+                ("ui", "hd"),
+                ('forceedit', 'captions'),
+                ("tab", "captions"),
+                ("o", "U")
+            )
+
+            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+
+        if not "accounts.google.com" in page.url:
+            break
+        else:
+            print("[Retrying in 30 seconds] Please supply authentication cookie information in config.json or environment variables. See README.md for more information.")
+            sleep(30)
+
+    inttext = page.text
 
-        filestring = "_community_draft"
-        
-        if '<li id="captions-editor-nav-captions" role="tab" data-state="published" class="published">' in inttext:
-            filestring = "_community_published"
+    try:
+        initlang = page.text.split("'metadataLanguage': \"", 1)[1].split('"', 1)[0]
+    except:
+        initlang = ""
 
-        if mode == "forceedit-captions":
-            filestring = "_community_draft"
+    del page
 
-        if 'title="The video owner already provided subtitles/CC"' in inttext:
-            filestring = "_uploader_provided"
+    filestring = "_community_draft"
+    
+    if '<li id="captions-editor-nav-captions" role="tab" data-state="published" class="published">' in inttext:
+        filestring = "_community_published"
 
-        if not "forceedit" in mode:
-            if '&amp;forceedit=metadata&amp;tab=metadata">See latest</a>' in inttext:
-                jobs.put((langcode, vid, "forceedit-metadata"))
-
-            if '<li id="captions-editor-nav-captions" role="tab" data-state="published" class="published">' in inttext:
-                jobs.put((langcode, vid, "forceedit-captions"))
-
-        if 'id="reject-captions-button"' in inttext or 'id="reject-metadata-button"' in inttext or 'data-state="published"' in inttext or 'title="The video owner already provided subtitles/CC"' in inttext: #quick way of checking if this page is worth parsing
-            parser = MyHTMLParser()
-            parser.feed(inttext)
-
-            captiontext = False
-            for item in parser.captions:
-                if item["text"][:-9]:
-                    captiontext = True
-
-            if captiontext and (mode == "default" or mode == "forceedit-captions"):
-                myfs = open("out/"+vid+"/"+vid+"_"+langcode+filestring+".sbv", "w", encoding="utf-8")
-                captions = parser.captions
-                captions.pop(0) #get rid of the fake one
-                while captions:
-                    item = captions.pop(0)
-
-                    myfs.write(timedelta_to_sbv_timestamp(timedelta(milliseconds=item["startTime"])) + "," + timedelta_to_sbv_timestamp(timedelta(milliseconds=item["endTime"])) + "\n" + item["text"][:-9] + "\n")
-                    
-                    del item
-                    if captions:
-                        myfs.write("\n")
-                del captions
-                myfs.close()
-                del myfs
-
-            del captiontext
-
-            if (parser.title or parser.description[:-16]) and (mode == "default" or mode == "forceedit-metadata"):
-                metadata = {}
-                metadata["title"] = parser.title
-                if metadata["title"] == False:
-                    metadata["title"] = ""
-                metadata["description"] = parser.description[:-16]
+    if mode == "forceedit-captions":
+        filestring = "_community_draft"
 
-                filestring = "_community_draft"
-                if '<li id="captions-editor-nav-metadata" role="tab" data-state="published" class="published">' in inttext:
-                    filestring = "_community_published"
+    if 'title="The video owner already provided subtitles/CC"' in inttext:
+        filestring = "_uploader_provided"
 
-                if mode == "forceedit-metadata":
-                    filestring = "_community_draft"
-                open("out/"+vid+"/"+vid+"_"+langcode+filestring+".json", "w", encoding="utf-8").write(dumps(metadata))
-                del metadata
+    if not "forceedit" in mode:
+        if '&amp;forceedit=metadata&amp;tab=metadata">See latest</a>' in inttext:
+            jobs.put(("subtitles-forceedit-metadata", vid, langcode))
 
-            if (parser.inittitle[9:-17] or parser.initdescription) and (mode == "default" or mode == "forceedit-metadata" and initlang):
-                metadata = {}
-                metadata["title"] = parser.inittitle[9:-17]
-                if metadata["title"] == False:
-                    metadata["title"] = ""
-                metadata["description"] = parser.initdescription
+        if '<li id="captions-editor-nav-captions" role="tab" data-state="published" class="published">' in inttext:
+            jobs.put(("subtitles-forceedit-captions", vid, langcode))
+
+    if 'id="reject-captions-button"' in inttext or 'id="reject-metadata-button"' in inttext or 'data-state="published"' in inttext or 'title="The video owner already provided subtitles/CC"' in inttext: #quick way of checking if this page is worth parsing
+        parser = MyHTMLParser()
+        parser.feed(inttext)
+
+        captiontext = False
+        for item in parser.captions:
+            if item["text"][:-9]:
+                captiontext = True
+
+        if captiontext and (mode == "default" or mode == "forceedit-captions"):
+            myfs = open("out/"+vid+"/"+vid+"_"+langcode+filestring+".sbv", "w", encoding="utf-8")
+            captions = parser.captions
+            captions.pop(0) #get rid of the fake one
+            while captions:
+                item = captions.pop(0)
+
+                myfs.write(timedelta_to_sbv_timestamp(timedelta(milliseconds=item["startTime"])) + "," + timedelta_to_sbv_timestamp(timedelta(milliseconds=item["endTime"])) + "\n" + item["text"][:-9] + "\n")
+                
+                del item
+                if captions:
+                    myfs.write("\n")
+            del captions
+            myfs.close()
+            del myfs
+
+        del captiontext
+
+        if (parser.title or parser.description[:-16]) and (mode == "default" or mode == "forceedit-metadata"):
+            metadata = {}
+            metadata["title"] = parser.title
+            if metadata["title"] == False:
+                metadata["title"] = ""
+            metadata["description"] = parser.description[:-16]
 
-                filestring = "_uploader_provided"
-                open("out/"+vid+"/"+vid+"_"+initlang+filestring+".json", "w", encoding="utf-8").write(dumps(metadata))
-                del metadata
+            filestring = "_community_draft"
+            if '<li id="captions-editor-nav-metadata" role="tab" data-state="published" class="published">' in inttext:
+                filestring = "_community_published"
 
-        del inttext
+            if mode == "forceedit-metadata":
+                filestring = "_community_draft"
+            open("out/"+vid+"/"+vid+"_"+langcode+filestring+".json", "w", encoding="utf-8").write(dumps(metadata))
+            del metadata
 
-        del langcode
-        del vid
-        del pparams
+        if (parser.inittitle[9:-17] or parser.initdescription) and (mode == "default" or mode == "forceedit-metadata" and initlang):
+            metadata = {}
+            metadata["title"] = parser.inittitle[9:-17]
+            if metadata["title"] == False:
+                metadata["title"] = ""
+            metadata["description"] = parser.initdescription
 
-        jobs.task_done()
+            filestring = "_uploader_provided"
+            open("out/"+vid+"/"+vid+"_"+initlang+filestring+".json", "w", encoding="utf-8").write(dumps(metadata))
+            del metadata
 
-    return True
+    del inttext
 
-if __name__ == "__main__":
-    from os import environ, mkdir
-    from os.path import isfile
-    from json import loads
-    #HSID, SSID, SID cookies required
-    if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():
-        cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}
-    elif isfile("config.json"):
-        cookies = loads(open("config.json").read())
-    else:
-        print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")
-        assert False
-    if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):
-        print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")
-        assert False
-
-    mysession = requests.session()
-    mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})
-    del cookies
-    from sys import argv
-    from queue import Queue
-    from threading import Thread
-    langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 
-    'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 
-    'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 
-    'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 
-    'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 
-    'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 
-    'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 
-    'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 
-    'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 
-    'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 
-    'xh', 'yi', 'yo', 'zu']
-    vidl = argv
-    vidl.pop(0)
+    del langcode
+    del vid
+    del pparams
 
-    try:
-        mkdir("out")
-    except:
-        pass
-
-    jobs = Queue()
-    for video in vidl:
-        try:
-            mkdir("out/"+video.strip())
-        except:
-            pass
-        for lang in langs:
-            jobs.put((lang, video, "default"))
-
-    subthreads = []
-
-    for r in range(50):
-        subrunthread = Thread(target=subprrun, args=(jobs,mysession))
-        subrunthread.start()
-        subthreads.append(subrunthread)
-        del subrunthread
-
-    for xa in subthreads:
-        xa.join() #bug (occurred once: the script ended before the last thread finished)
-        subthreads.remove(xa)
-        del xa
\ No newline at end of file
+    return True
+
+# if __name__ == "__main__":
+#     from os import environ, mkdir
+#     from os.path import isfile
+#     from json import loads
+#     #HSID, SSID, SID cookies required
+#     if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():
+#         cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}
+#     elif isfile("config.json"):
+#         cookies = loads(open("config.json").read())
+#     else:
+#         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")
+#         assert False
+#     if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):
+#         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")
+#         assert False
+
+#     mysession = requests.session()
+#     mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})
+#     del cookies
+#     from sys import argv
+#     from queue import Queue
+#     from threading import Thread
+#     langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 
+#     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 
+#     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 
+#     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 
+#     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 
+#     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 
+#     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 
+#     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 
+#     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 
+#     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 
+#     'xh', 'yi', 'yo', 'zu']
+#     vidl = argv
+#     vidl.pop(0)
+
+#     try:
+#         mkdir("out")
+#     except:
+#         pass
+
+#     jobs = Queue()
+#     for video in vidl:
+#         try:
+#             mkdir("out/"+video.strip())
+#         except:
+#             pass
+#         for lang in langs:
+#             jobs.put((lang, video, "default"))
+
+#     subthreads = []
+
+#     for r in range(50):
+#         subrunthread = Thread(target=subprrun, args=(jobs,mysession))
+#         subrunthread.start()
+#         subthreads.append(subrunthread)
+#         del subrunthread
+
+#     for xa in subthreads:
+#         xa.join() #bug (occurred once: the script ended before the last thread finished)
+#         subthreads.remove(xa)
+#         del xa
\ No newline at end of file
diff --git a/worker.py b/worker.py
index 08b0915..5be8086 100644
--- a/worker.py
+++ b/worker.py
@@ -87,6 +87,8 @@ class GracefulKiller:
 
 gkiller = GracefulKiller()
 
+#TODO: discoveries, zipping, completion of subtitles
+
 #minitasks
 def threadrunner(jobs: Queue):
     global langcnt
@@ -98,9 +100,35 @@ def threadrunner(jobs: Queue):
             if task == "submitdiscovery":
                 tracker.add_item_to_tracker(args, vid)
             elif task == "discovery":
-                pass
+                while True:
+                    try:
+                        info = getmetadata(mysession, str(vid).strip())
+                        break
+                    except BaseException as e:
+                        print(e)
+                        print("Error in retrieving information, waiting 30 seconds")
+                        sleep(30)
+                if info[0] or info[1]: # ccenabled or creditdata
+                    if not isdir("out/"+str(vid).strip()):
+                        mkdir("out/"+str(vid).strip())
+                if info[1]:
+                    open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))
+
+                if info[0]:
+                    langcnt[vid] = 0
+                    for langcode in langs:
+                        jobs.put(("subtitles", vid, langcode))
+                else:
+                    jobs.put(("complete", None, "video:"+vid))
             elif task == "subtitles":
-                pass
+                subprrun(jobs, mysession, args, vid, "default")
+                langcnt[vid] += 1
+                if langcnt[vid] >= 195:
+                    pass #complete(?)
+            elif task == "subtitles-forceedit-captions":
+                subprrun(jobs, mysession, args, vid, "forceedit-captions")
+            elif task == "subtitles-forceedit-metadata":
+                subprrun(jobs, mysession, args, vid, "forceedit-metadata")
             elif task == "channel":
                 y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
                 for itemyv in y["entries"]:
@@ -185,14 +213,7 @@ def prrun():
         item = jobs.get()
 
         print("Video ID:", str(item).strip())
-        while True:
-            try:
-                info = getmetadata(mysession, str(item).strip())
-                break
-            except BaseException as e:
-                print(e)
-                print("Error in retrieving information, waiting 30 seconds")
-                sleep(30)
+        
 
         # Add any discovered videos
         recvids.update(info[2])
@@ -200,10 +221,6 @@ def prrun():
         recmixes.update(info[4])
         recplayl.update(info[5])
 
-        if info[0] or info[1]: # ccenabled or creditdata
-            if not isdir("out/"+str(item).strip()):
-                mkdir("out/"+str(item).strip())
-
         if info[1]: # creditdata
             open("out/"+str(item).strip()+"/"+str(item).strip()+"_published_credits.json", "w").write(dumps(info[1]))
 

From a96d4c7b4846c98c45254846642f79fcc05155d5 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 21:55:13 -0400
Subject: [PATCH 03/13] More WIP

---
 worker.py | 229 +++++++++++-------------------------------------------
 1 file changed, 45 insertions(+), 184 deletions(-)

diff --git a/worker.py b/worker.py
index 5be8086..3289b51 100644
--- a/worker.py
+++ b/worker.py
@@ -25,6 +25,15 @@ jobs = Queue()
 
 langcnt = {}
 
+try:
+    mkdir("out")
+except:
+    pass
+
+try:
+    mkdir("directory")
+except:
+    pass
 
 HEROKU = False
 if isfile("../Procfile"):
@@ -87,9 +96,9 @@ class GracefulKiller:
 
 gkiller = GracefulKiller()
 
-#TODO: discoveries, zipping, completion of subtitles
+#TODO: zipping, completion of subtitles (return value), limit task retrieval count
 
-#minitasks
+#microtasks
 def threadrunner(jobs: Queue):
     global langcnt
     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})
@@ -120,6 +129,16 @@ def threadrunner(jobs: Queue):
                         jobs.put(("subtitles", vid, langcode))
                 else:
                     jobs.put(("complete", None, "video:"+vid))
+
+                for videodisc in info[2]:
+                    jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))
+                for channeldisc in info[3]:
+                    jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))
+                for mixdisc in info[4]:
+                    jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))
+                for playldisc in info[5]:
+                    jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))
+
             elif task == "subtitles":
                 subprrun(jobs, mysession, args, vid, "default")
                 langcnt[vid] += 1
@@ -143,11 +162,32 @@ def threadrunner(jobs: Queue):
                 size = 0
                 if ":" in args:
                     if args.split(":", 1)[0] == "video":
+                        #check if dir is empty, make zip if needed
                         if isfile("directory/"+args.split(":", 1)[1]+".zip"):
                             size = getsize("directory/"+args.split(":", 1)[1]+".zip")
-                tracker.mark_item_as_done(args, size)
+
+                            #get a target
+                            targetloc = None
+                            while not targetloc:
+                                targetloc = tracker.request_upload_target()
+                                if targetloc:
+                                    break
+                                else:
+                                    print("Waiting 5 minutes...")
+                                    sleep(300)
+
+                            if targetloc.startswith("rsync"):
+                                system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/ "+targetloc)
+                            elif targetloc.startswith("http"):
+                                for filzip in listdir("directory"):
+                                    if filzip.endswith(".zip"):
+                                        system("curl -F "+filzip+"=@directory/"+filzip+" "+targetloc)
+                                        tracker.mark_item_as_done(args, size)
+
+            jobs.task_done()
         else:
             # get a new task from tracker
+            collect() #cleanup
             desit = tracker.request_item_from_tracker()
             if desit:
                 if desit.split(":", 1)[0] == "video":
@@ -160,107 +200,14 @@ def threadrunner(jobs: Queue):
                     print("Ignoring item for now", desit)
             else:
                 print("Ignoring item for now", desit)
-            
-            batchcontent.append(desit.split(":", 1)[1])
-            actualitems.append(desit)
-
-
-
-
-batchcontent = []
-actualitems = []
-
-
-def batchfunc():
-    
-    
-    if not HEROKU:
-        desqsize = 51
-    elif HEROKU:
-        desqsize = 251
-    
-    while jobs.qsize() < desqsize:
-
-
-def submitfunc(submitqueue):
-    while not submitqueue.empty():
-        itype, ival = submitqueue.get()
-        tracker.add_item_to_tracker(itype, ival)
-
-
-
-
-
-ccenabledl = []
-
-recvids  = set()
-recchans = set()
-recmixes = set()
-recplayl = set()
-
-
-
-
 
 def prrun():
-    while not jobs.empty():
-        global recvids
-        global recchans
-        global recmixes
-        global recplayl
-        global ccenabledl
-
-        item = jobs.get()
 
         print("Video ID:", str(item).strip())
         
 
-        # Add any discovered videos
-        recvids.update(info[2])
-        recchans.update(info[3])
-        recmixes.update(info[4])
-        recplayl.update(info[5])
-
-        if info[1]: # creditdata
-            open("out/"+str(item).strip()+"/"+str(item).strip()+"_published_credits.json", "w").write(dumps(info[1]))
-
-        if info[0]: #ccenabled
-            ccenabledl.append(item)
-        jobs.task_done()
-
-    return True
-
 while not gkiller.kill_now:
-    collect() #cleanup
-
-    try:
-        mkdir("out")
-    except:
-        pass
-
-    try:
-        mkdir("directory")
-    except:
-        pass
-
-    batchcontent.clear()
-    actualitems.clear()
-
-    # Get a batch ID
-    batchthreads = []
-
-    for r in range(50):
-        batchrunthread = Thread(target=batchfunc)
-        batchrunthread.start()
-        batchthreads.append(batchrunthread)
-        del batchrunthread
-
-    for xc in batchthreads:
-        xc.join()
-        batchthreads.remove(xc)
-        del xc
-
-    sleep(1) # prevent the script from continuing before the last thread finishes
+    
 
     threads = []
 
@@ -275,76 +222,6 @@ while not gkiller.kill_now:
         threads.remove(x)
         del x
 
-    print("Sending discoveries to tracker...")
-
-    submitjobs = Queue()
-
-    # IDK how to handle mixes so just send them for now
-    print("Videos:", len(recvids))
-    for itemvid in recvids:
-        submitjobs.put((tracker.ItemType.Video, itemvid))
-
-    print("Channels:", len(recchans))
-    for itemchan in recchans:
-        submitjobs.put((tracker.ItemType.Channel, itemchan))
-
-    print("Mix Playlists:", len(recmixes))
-    for itemmix in recmixes:
-        submitjobs.put((tracker.ItemType.MixPlaylist, itemmix))
-
-    print("Playlists:", len(recplayl))
-    for itemplayl in recplayl:
-        submitjobs.put((tracker.ItemType.Playlist, itemplayl))
-
-    # open("out/discoveries.json", "w").write(dumps({"recvids": sorted(recvids), "recchans": sorted(recchans), "recmixes": sorted(recmixes), "recplayl": sorted(recplayl)}))
-    
-    # clear lists
-    recvids.clear()
-    recchans.clear()
-    recmixes.clear()
-    recplayl.clear()
-
-    submitthreads = []
-
-    for r in range(50):
-        submitrunthread = Thread(target=submitfunc, args=(submitjobs,))
-        submitrunthread.start()
-        submitthreads.append(submitrunthread)
-        del submitrunthread
-
-    for xb in submitthreads:
-        xb.join()
-        submitthreads.remove(xb)
-        del xb
-
-    sleep(1) # prevent the script from continuing before the last thread finishes
-
-
-    subtjobs = Queue()
-    while ccenabledl:
-        langcontent = langs.copy()
-        intvid = ccenabledl.pop(0)
-
-        while langcontent:
-            subtjobs.put((langcontent.pop(0), intvid, "default"))
-        del intvid
-        del langcontent
-
-    subthreads = []
-
-    for r in range(50):
-        subrunthread = Thread(target=subprrun, args=(subtjobs,mysession))
-        subrunthread.start()
-        subthreads.append(subrunthread)
-        del subrunthread
-
-    for xa in subthreads:
-        xa.join()
-        subthreads.remove(xa)
-        del xa
-
-    sleep(30) # wait 30 seconds to hopefully allow the other threads to finish
-
     for fol in listdir("out"): #remove empty folders
         try:
             if isdir("out/"+fol):
@@ -359,26 +236,10 @@ while not gkiller.kill_now:
         if isdir("out/"+fol):
             make_archive("directory/"+fol, "zip", "out/"+fol)
 
-    targetloc = None
-    while not targetloc:
-        targetloc = tracker.request_upload_target()
-        if targetloc:
-            break
-        else:
-            print("Waiting 5 minutes...")
-            sleep(300)
 
-    if targetloc.startswith("rsync"):
-        system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/ "+targetloc)
-    elif targetloc.startswith("http"):
-        for filzip in listdir("directory"):
-            if filzip.endswith(".zip"):
-                system("curl -F "+filzip+"=@directory/"+filzip+" "+targetloc)
 
-    # Report the batch as complete
-    for itemb in actualitems:
 
-        tracker.mark_item_as_done(itemb, size)
+
 
     # clear the output directories
     rmtree("out")

From 6fcdd75ada59b1e35be9cebd5b70a6031f9b7ab1 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:34:42 -0400
Subject: [PATCH 04/13] More updates

---
 export.py  |  11 +++-
 tracker.py |   2 +-
 worker.py  | 163 ++++++++++++++++++++++++++++-------------------------
 3 files changed, 97 insertions(+), 79 deletions(-)

diff --git a/export.py b/export.py
index 76053dd..c8f2d03 100644
--- a/export.py
+++ b/export.py
@@ -31,7 +31,7 @@ from json import dumps
 
 from gc import collect
 
-import requests
+# import requests
 
 from time import sleep
 
@@ -81,6 +81,11 @@ class MyHTMLParser(HTMLParser):
 
 def subprrun(jobs, mysession, langcode, vid, mode):
     collect() #cleanup memory
+
+    if not "forceedit" in mode:
+        retval = 3
+    else:
+        retval = 1
     vid = vid.strip()
     print(langcode, vid)
 
@@ -150,9 +155,11 @@ def subprrun(jobs, mysession, langcode, vid, mode):
     if not "forceedit" in mode:
         if '&amp;forceedit=metadata&amp;tab=metadata">See latest</a>' in inttext:
             jobs.put(("subtitles-forceedit-metadata", vid, langcode))
+            retval -= 1
 
         if '<li id="captions-editor-nav-captions" role="tab" data-state="published" class="published">' in inttext:
             jobs.put(("subtitles-forceedit-captions", vid, langcode))
+            retval -= 1
 
     if 'id="reject-captions-button"' in inttext or 'id="reject-metadata-button"' in inttext or 'data-state="published"' in inttext or 'title="The video owner already provided subtitles/CC"' in inttext: #quick way of checking if this page is worth parsing
         parser = MyHTMLParser()
@@ -214,7 +221,7 @@ def subprrun(jobs, mysession, langcode, vid, mode):
     del vid
     del pparams
 
-    return True
+    return retval
 
 # if __name__ == "__main__":
 #     from os import environ, mkdir
diff --git a/tracker.py b/tracker.py
index 90da312..c2abd7a 100644
--- a/tracker.py
+++ b/tracker.py
@@ -9,7 +9,7 @@ from os.path import isfile
 from json import loads
 
 # https://github.com/ArchiveTeam/tencent-weibo-grab/blob/9bae5f9747e014db9227821a9c11557267967023/pipeline.py
-VERSION = "20200923.02"
+VERSION = "20200923.03"
 
 TRACKER_ID = "ext-yt-communitycontribs"
 TRACKER_HOST = "trackerproxy.meo.ws"
diff --git a/worker.py b/worker.py
index 3289b51..397d065 100644
--- a/worker.py
+++ b/worker.py
@@ -11,12 +11,14 @@ import tracker
 
 from youtube_dl import YoutubeDL
 
-from shutil import make_archive, rmtree
+from shutil import rmtree
 
 from queue import Queue
 
 from gc import collect
 
+from datetime import timedelta, datetime
+
 from discovery import getmetadata
 from export import subprrun
 
@@ -25,6 +27,8 @@ jobs = Queue()
 
 langcnt = {}
 
+lasttask = datetime.min()
+
 try:
     mkdir("out")
 except:
@@ -91,16 +95,16 @@ class GracefulKiller:
         signal.signal(signal.SIGINT, self.exit_gracefully)
         signal.signal(signal.SIGTERM, self.exit_gracefully)
 
-    def exit_gracefully(self,signum, frame):
+    def exit_gracefully(self, signum, frame):
+        print("Graceful exit process initiated, stopping all tasks...")
         self.kill_now = True
 
 gkiller = GracefulKiller()
 
-#TODO: zipping, completion of subtitles (return value), limit task retrieval count
-
 #microtasks
 def threadrunner(jobs: Queue):
     global langcnt
+    global lasttask
     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})
     while not gkiller.kill_now:
         if not jobs.empty():
@@ -140,10 +144,10 @@ def threadrunner(jobs: Queue):
                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))
 
             elif task == "subtitles":
-                subprrun(jobs, mysession, args, vid, "default")
-                langcnt[vid] += 1
-                if langcnt[vid] >= 195:
-                    pass #complete(?)
+                retval = subprrun(jobs, mysession, args, vid, "default")
+                langcnt[vid] += retval
+                if langcnt[vid] >= 585:
+                    jobs.put(("complete", None, "video:"+vid))
             elif task == "subtitles-forceedit-captions":
                 subprrun(jobs, mysession, args, vid, "forceedit-captions")
             elif task == "subtitles-forceedit-metadata":
@@ -163,84 +167,91 @@ def threadrunner(jobs: Queue):
                 if ":" in args:
                     if args.split(":", 1)[0] == "video":
                         #check if dir is empty, make zip if needed
-                        if isfile("directory/"+args.split(":", 1)[1]+".zip"):
-                            size = getsize("directory/"+args.split(":", 1)[1]+".zip")
-
-                            #get a target
-                            targetloc = None
-                            while not targetloc:
-                                targetloc = tracker.request_upload_target()
-                                if targetloc:
-                                    break
-                                else:
-                                    print("Waiting 5 minutes...")
-                                    sleep(300)
-
-                            if targetloc.startswith("rsync"):
-                                system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/ "+targetloc)
-                            elif targetloc.startswith("http"):
-                                for filzip in listdir("directory"):
-                                    if filzip.endswith(".zip"):
-                                        system("curl -F "+filzip+"=@directory/"+filzip+" "+targetloc)
-                                        tracker.mark_item_as_done(args, size)
-
+                        if isdir("out/"+args.split(":", 1)[1]):
+                            if not listdir("out/"+args.split(":", 1)[1]):
+                                rmdir("out/"+args.split(":", 1)[1])
+                            else:
+                                #zip it up
+                                if not isdir("directory/"+args.split(":", 1)[1]):
+                                    mkdir("directory/"+args.split(":", 1)[1])
+
+                                while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):
+                                    print("Attempting to zip item...")
+                                    system("zip -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])
+
+                                #get a target
+                                targetloc = None
+                                while not targetloc:
+                                    targetloc = tracker.request_upload_target()
+                                    if targetloc:
+                                        break
+                                    else:
+                                        print("Waiting 5 minutes...")
+                                        sleep(300)
+
+                                if targetloc.startswith("rsync"):
+                                    system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/"+args.split(":", 1)[1]+"/ "+targetloc)
+                                elif targetloc.startswith("http"):
+                                    system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)
+
+
+                                size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")
+                                #cleanup
+                                try:
+                                    del langcnt[args.split(":", 1)[1]]
+                                    rmtree("directory/"+args.split(":", 1)[1]+"/")
+                                    rmdir("directory/"+args.split(":", 1)[1]+"/")
+                                    rmtree("out/"+args.split(":", 1)[1]+"/")
+                                    rmdir("out/"+args.split(":", 1)[1]+"/")
+                                except:
+                                    pass
+                tracker.mark_item_as_done(args, size)
             jobs.task_done()
         else:
             # get a new task from tracker
-            collect() #cleanup
-            desit = tracker.request_item_from_tracker()
-            if desit:
-                if desit.split(":", 1)[0] == "video":
-                    jobs.put(("discovery", desit.split(":", 1)[1], None))
-                elif desit.split(":", 1)[0] == "channel":
-                    jobs.put(("channel", None, desit.split(":", 1)[1]))
-                elif desit.split(":", 1)[0] == "playlist":
-                    jobs.put(("playlist", None, desit.split(":", 1)[1]))
+            if datetime.now() - lasttask > timedelta(seconds=15): #only retrieve a task every 15 seconds to allow queue to build up
+                collect() #cleanup
+                desit = tracker.request_item_from_tracker()
+                print("New task:", desit)
+                if desit:
+                    if desit.split(":", 1)[0] == "video":
+                        lasttask = datetime.now()
+                        jobs.put(("discovery", desit.split(":", 1)[1], None))
+                    elif desit.split(":", 1)[0] == "channel":
+                        lasttask = datetime.now()
+                        jobs.put(("channel", None, desit.split(":", 1)[1]))
+                    elif desit.split(":", 1)[0] == "playlist":
+                        lasttask = datetime.now()
+                        jobs.put(("playlist", None, desit.split(":", 1)[1]))
+                    else:
+                        print("Ignoring item for now", desit)
                 else:
                     print("Ignoring item for now", desit)
             else:
-                print("Ignoring item for now", desit)
-
-def prrun():
-
-        print("Video ID:", str(item).strip())
-        
-
-while not gkiller.kill_now:
+                sleep(1)
     
 
-    threads = []
-
-    for i in range(50):
-        runthread = Thread(target=prrun)
-        runthread.start()
-        threads.append(runthread)
-        del runthread
-
-    for x in threads:
-        x.join()
-        threads.remove(x)
-        del x
-
-    for fol in listdir("out"): #remove empty folders
-        try:
-            if isdir("out/"+fol):
-                rmdir("out/"+fol)
-        except:
-            pass
-
-    #https://stackoverflow.com/a/11968881
-
-
-    for fol in listdir("out"):
-        if isdir("out/"+fol):
-            make_archive("directory/"+fol, "zip", "out/"+fol)
-
+threads = []
 
+#start with 1 thread, give it a 5 second head start
+runthread = Thread(target=threadrunner, args=(jobs,))
+runthread.start()
+threads.append(runthread)
+del runthread
 
+sleep(5)
 
+#now create the other 49 threads
+for i in range(49):
+    runthread = Thread(target=threadrunner, args=(jobs,))
+    runthread.start()
+    threads.append(runthread)
+    del runthread
 
+#https://stackoverflow.com/a/11968881
+for x in threads:
+    x.join()
+    threads.remove(x)
+    del x
 
-    # clear the output directories
-    rmtree("out")
-    rmtree("directory")
\ No newline at end of file
+print("Exiting...")
\ No newline at end of file

From e4a4301346e010c1804e6a54b172d2bc98738f91 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:37:29 -0400
Subject: [PATCH 05/13] Bug fix

---
 worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/worker.py b/worker.py
index 397d065..a15e3f7 100644
--- a/worker.py
+++ b/worker.py
@@ -27,7 +27,7 @@ jobs = Queue()
 
 langcnt = {}
 
-lasttask = datetime.min()
+lasttask = datetime.min
 
 try:
     mkdir("out")

From 98f96792ed97a28e4a1b0d7e473b42040cfdde1b Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:42:50 -0400
Subject: [PATCH 06/13] Small adjustment

---
 worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/worker.py b/worker.py
index a15e3f7..6fb14eb 100644
--- a/worker.py
+++ b/worker.py
@@ -177,7 +177,7 @@ def threadrunner(jobs: Queue):
 
                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):
                                     print("Attempting to zip item...")
-                                    system("zip -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])
+                                    system("zip -9 -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])
 
                                 #get a target
                                 targetloc = None

From 4d5d1bf9a3dd44dd9c5928c313fb846a0ebea654 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:54:39 -0400
Subject: [PATCH 07/13] Small adjustments

---
 tracker.py | 2 +-
 worker.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tracker.py b/tracker.py
index c2abd7a..c457ea3 100644
--- a/tracker.py
+++ b/tracker.py
@@ -9,7 +9,7 @@ from os.path import isfile
 from json import loads
 
 # https://github.com/ArchiveTeam/tencent-weibo-grab/blob/9bae5f9747e014db9227821a9c11557267967023/pipeline.py
-VERSION = "20200923.03"
+VERSION = "20200923.04"
 
 TRACKER_ID = "ext-yt-communitycontribs"
 TRACKER_HOST = "trackerproxy.meo.ws"
diff --git a/worker.py b/worker.py
index 6fb14eb..101cbdb 100644
--- a/worker.py
+++ b/worker.py
@@ -177,7 +177,7 @@ def threadrunner(jobs: Queue):
 
                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):
                                     print("Attempting to zip item...")
-                                    system("zip -9 -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])
+                                    system("zip -9 -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1]+"/*")
 
                                 #get a target
                                 targetloc = None

From e7d921d3f7776ac7cf8abc8b140a662433c4bf30 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:57:46 -0400
Subject: [PATCH 08/13] Handle youtube-dl errors

---
 worker.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/worker.py b/worker.py
index 101cbdb..31d745c 100644
--- a/worker.py
+++ b/worker.py
@@ -153,12 +153,24 @@ def threadrunner(jobs: Queue):
             elif task == "subtitles-forceedit-metadata":
                 subprrun(jobs, mysession, args, vid, "forceedit-metadata")
             elif task == "channel":
-                y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
+                while True:
+                    try:
+                        y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
+                        break
+                    except:
+                        print("YouTube-DL error, waiting 30 seconds...")
+                        sleep(30)
                 for itemyv in y["entries"]:
                     jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))
                 jobs.put(("complete", None, "channel:"+args))
             elif task == "playlist":
-                y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)
+                while True:
+                    try:
+                        y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)
+                        break
+                    except:
+                        print("YouTube-DL error, waiting 30 seconds...")
+                        sleep(30)
                 for itemyvp in y["entries"]:
                     jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))
                 jobs.put(("complete", None, "playlist:"+args))

From 82f52c241b34ca97c415ef62430c9b963a91d58f Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 22:59:42 -0400
Subject: [PATCH 09/13] Ensure needed commands are installed

---
 worker.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/worker.py b/worker.py
index 31d745c..a686a7e 100644
--- a/worker.py
+++ b/worker.py
@@ -11,7 +11,7 @@ import tracker
 
 from youtube_dl import YoutubeDL
 
-from shutil import rmtree
+from shutil import rmtree, which
 
 from queue import Queue
 
@@ -55,6 +55,8 @@ langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az
     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 
     'xh', 'yi', 'yo', 'zu']
 
+assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."
+
 #HSID, SSID, SID cookies required
 if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():
     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

From d0ef6eeae2cc2538a96ce3ba3fa7111612c29220 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 23:04:25 -0400
Subject: [PATCH 10/13] Improve youtube-dl handling

---
 tracker.py |  2 +-
 worker.py  | 18 ++++++++----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/tracker.py b/tracker.py
index c457ea3..b5bd685 100644
--- a/tracker.py
+++ b/tracker.py
@@ -9,7 +9,7 @@ from os.path import isfile
 from json import loads
 
 # https://github.com/ArchiveTeam/tencent-weibo-grab/blob/9bae5f9747e014db9227821a9c11557267967023/pipeline.py
-VERSION = "20200923.04"
+VERSION = "20200923.05"
 
 TRACKER_ID = "ext-yt-communitycontribs"
 TRACKER_HOST = "trackerproxy.meo.ws"
diff --git a/worker.py b/worker.py
index a686a7e..3454033 100644
--- a/worker.py
+++ b/worker.py
@@ -158,24 +158,22 @@ def threadrunner(jobs: Queue):
                 while True:
                     try:
                         y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)
+                        for itemyv in y["entries"]:
+                            jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))
+                        jobs.put(("complete", None, "channel:"+args))
                         break
                     except:
-                        print("YouTube-DL error, waiting 30 seconds...")
-                        sleep(30)
-                for itemyv in y["entries"]:
-                    jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))
-                jobs.put(("complete", None, "channel:"+args))
+                        print("YouTube-DL error, ignoring but not marking as complete...")
             elif task == "playlist":
                 while True:
                     try:
                         y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)
+                        for itemyvp in y["entries"]:
+                            jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))
+                        jobs.put(("complete", None, "playlist:"+args))
                         break
                     except:
-                        print("YouTube-DL error, waiting 30 seconds...")
-                        sleep(30)
-                for itemyvp in y["entries"]:
-                    jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))
-                jobs.put(("complete", None, "playlist:"+args))
+                        print("YouTube-DL error, ignoring but not marking as complete...")
             elif task == "complete":
                 size = 0
                 if ":" in args:

From 3d72edc11ff3b86fe33dfdc91b4ae7897e26050b Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 23:16:12 -0400
Subject: [PATCH 11/13] Fix zip generation

---
 tracker.py | 2 +-
 worker.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tracker.py b/tracker.py
index b5bd685..480eee3 100644
--- a/tracker.py
+++ b/tracker.py
@@ -9,7 +9,7 @@ from os.path import isfile
 from json import loads
 
 # https://github.com/ArchiveTeam/tencent-weibo-grab/blob/9bae5f9747e014db9227821a9c11557267967023/pipeline.py
-VERSION = "20200923.05"
+VERSION = "20200923.06"
 
 TRACKER_ID = "ext-yt-communitycontribs"
 TRACKER_HOST = "trackerproxy.meo.ws"
diff --git a/worker.py b/worker.py
index 3454033..c810449 100644
--- a/worker.py
+++ b/worker.py
@@ -189,7 +189,7 @@ def threadrunner(jobs: Queue):
 
                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):
                                     print("Attempting to zip item...")
-                                    system("zip -9 -r directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1]+"/*")
+                                    system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])
 
                                 #get a target
                                 targetloc = None

From 984c42db5347053cff4ed9551bcb215324e58074 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 23:21:43 -0400
Subject: [PATCH 12/13] youtube-dl error logging

---
 worker.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/worker.py b/worker.py
index c810449..9911b7a 100644
--- a/worker.py
+++ b/worker.py
@@ -163,7 +163,7 @@ def threadrunner(jobs: Queue):
                         jobs.put(("complete", None, "channel:"+args))
                         break
                     except:
-                        print("YouTube-DL error, ignoring but not marking as complete...")
+                        print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])
             elif task == "playlist":
                 while True:
                     try:
@@ -173,7 +173,7 @@ def threadrunner(jobs: Queue):
                         jobs.put(("complete", None, "playlist:"+args))
                         break
                     except:
-                        print("YouTube-DL error, ignoring but not marking as complete...")
+                        print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])
             elif task == "complete":
                 size = 0
                 if ":" in args:

From dac703aafaf393a1cfe25b0e0d5ed4b9bc2f0751 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Wed, 23 Sep 2020 23:29:07 -0400
Subject: [PATCH 13/13] Export retries on exception

---
 export.py  | 86 ++++++++++++++++++++++++++++--------------------------
 tracker.py |  2 +-
 2 files changed, 46 insertions(+), 42 deletions(-)

diff --git a/export.py b/export.py
index c8f2d03..d8bec63 100644
--- a/export.py
+++ b/export.py
@@ -90,47 +90,51 @@ def subprrun(jobs, mysession, langcode, vid, mode):
     print(langcode, vid)
 
     while True:
-        if mode == "default":
-            pparams = (
-                ("v", vid),
-                ("lang", langcode),
-                ("action_mde_edit_form", 1),
-                ("bl", "vmp"),
-                ("ui", "hd"),
-                ("tab", "captions"),
-                ("o", "U")
-            )
-
-            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-        elif mode == "forceedit-metadata":
-            pparams = (
-                ("v", vid),
-                ("lang", langcode),
-                ("action_mde_edit_form", 1),
-                ('forceedit', 'metadata'),
-                ('tab', 'metadata')
-            )
-
-            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-        elif mode == "forceedit-captions":
-            pparams = (
-                ("v", vid),
-                ("lang", langcode),
-                ("action_mde_edit_form", 1),
-                ("bl", "vmp"),
-                ("ui", "hd"),
-                ('forceedit', 'captions'),
-                ("tab", "captions"),
-                ("o", "U")
-            )
-
-            page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
-
-        if not "accounts.google.com" in page.url:
-            break
-        else:
-            print("[Retrying in 30 seconds] Please supply authentication cookie information in config.json or environment variables. See README.md for more information.")
-            sleep(30)
+        try:
+            if mode == "default":
+                pparams = (
+                    ("v", vid),
+                    ("lang", langcode),
+                    ("action_mde_edit_form", 1),
+                    ("bl", "vmp"),
+                    ("ui", "hd"),
+                    ("tab", "captions"),
+                    ("o", "U")
+                )
+
+                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+            elif mode == "forceedit-metadata":
+                pparams = (
+                    ("v", vid),
+                    ("lang", langcode),
+                    ("action_mde_edit_form", 1),
+                    ('forceedit', 'metadata'),
+                    ('tab', 'metadata')
+                )
+
+                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+            elif mode == "forceedit-captions":
+                pparams = (
+                    ("v", vid),
+                    ("lang", langcode),
+                    ("action_mde_edit_form", 1),
+                    ("bl", "vmp"),
+                    ("ui", "hd"),
+                    ('forceedit', 'captions'),
+                    ("tab", "captions"),
+                    ("o", "U")
+                )
+
+                page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams)
+
+            if not "accounts.google.com" in page.url:
+                break
+            else:
+                print("[Retrying in 30 seconds] Please supply authentication cookie information in config.json or environment variables. See README.md for more information.")
+                sleep(30)
+        except:
+            print("Error in request, retrying in 5 seconds...")
+            sleep(5)
 
     inttext = page.text
 
diff --git a/tracker.py b/tracker.py
index 480eee3..38aad30 100644
--- a/tracker.py
+++ b/tracker.py
@@ -9,7 +9,7 @@ from os.path import isfile
 from json import loads
 
 # https://github.com/ArchiveTeam/tencent-weibo-grab/blob/9bae5f9747e014db9227821a9c11557267967023/pipeline.py
-VERSION = "20200923.06"
+VERSION = "20200923.07"
 
 TRACKER_ID = "ext-yt-communitycontribs"
 TRACKER_HOST = "trackerproxy.meo.ws"