diff --git a/export.py b/export.py index c89437c..26f1d38 100644 --- a/export.py +++ b/export.py @@ -25,26 +25,17 @@ def timedelta_to_sbv_timestamp(timedelta_timestamp): return "%1d:%02d:%02d.%03d" % (hrs, mins, secs, msecs) -import requests from bs4 import BeautifulSoup from datetime import timedelta -import threading - -from os import mkdir - -from json import loads, dumps - - -#HSID, SSID, SID cookies required -cookies = loads(open("config.json").read()) -mysession = requests.session() -mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",}) +from json import dumps -class subtitlethread(threading.Thread): - def run(self): - langcode, vid = self.getName().split(";", 1) +import requests +def subprrun(jobs, headers): + while not jobs.empty(): + langcode, vid = jobs.get() + print(langcode, vid) pparams = ( ("v", vid), ("lang", langcode), @@ -55,24 +46,32 @@ class subtitlethread(threading.Thread): ("o", "U") ) - page = mysession.get("https://www.youtube.com/timedtext_editor", params=pparams) + page = requests.get("https://www.youtube.com/timedtext_editor", headers=headers, params=pparams) assert not "accounts.google.com" in page.url, "Please supply authentication cookie information in config.json. See README.md for more information." soup = BeautifulSoup(page.text, features="html5lib") + del page divs = soup.find_all("div", class_="timed-event-line") - outtext = "" - - for item in divs: + myfs = open("out/"+vid+"/"+vid+"_"+langcode+".sbv", "w", encoding="utf-8") + while divs: + item = divs.pop(0) text = item.find("textarea").text startms = int(item.find("input", class_="event-start-time")["data-start-ms"]) endms = int(item.find("input", class_="event-end-time")["data-end-ms"]) - outtext += timedelta_to_sbv_timestamp(timedelta(milliseconds=startms)) + "," + timedelta_to_sbv_timestamp(timedelta(milliseconds=endms)) + "\n" + text + "\n\n" - - open("out/"+vid+"/"+vid+"_"+langcode+".sbv", "w", encoding="utf-8").write(outtext[:-1]) + myfs.write(timedelta_to_sbv_timestamp(timedelta(milliseconds=startms)) + "," + timedelta_to_sbv_timestamp(timedelta(milliseconds=endms)) + "\n" + text + "\n") + + del item + del text + del startms + del endms + if divs: + myfs.write("\n") + del divs + del myfs if soup.find("li", id="captions-editor-nav-metadata")["data-state"] != "locked": metadata = {} @@ -84,42 +83,13 @@ class subtitlethread(threading.Thread): metadata["description"] = soup.find("textarea", id="metadata-description").text open("out/"+vid+"/"+vid+"_"+langcode+".json", "w", encoding="utf-8").write(dumps(metadata)) + del metadata + + del soup + del langcode + del vid + del pparams + + jobs.task_done() -def getsubs(vid): - langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', - 'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', - 'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', - 'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', - 'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', - 'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', - 'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', - 'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', - 'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', - 'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', - 'xh', 'yi', 'yo', 'zu'] - - threads = [] - for langcode in langs: - runthread = subtitlethread(name = langcode+";"+vid) - runthread.start() - threads.append(runthread) - - for x in threads: - x.join() - - return True - -if __name__ == "__main__": - from sys import argv - vidl = argv - vidl.pop(0) - for video in vidl: - try: - mkdir("out") - except: - pass - try: - mkdir("out/"+video) - except: - pass - getsubs(video) \ No newline at end of file + return True \ No newline at end of file diff --git a/worker.py b/worker.py index d1ae540..384b98f 100644 --- a/worker.py +++ b/worker.py @@ -1,25 +1,57 @@ +from threading import Thread import requests from time import sleep from os import mkdir from os.path import isdir -from json import dumps -import threading +from json import dumps, loads from shutil import make_archive, rmtree +from queue import Queue + from discovery import getmetadata -from export import getsubs +from export import subprrun WORKER_VERSION = 1 SERVER_BASE_URL = "http://localhost:5000" -class batchthread(threading.Thread): - def run(self): - item = self.getName() +langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', + 'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', + 'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', + 'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', + 'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', + 'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', + 'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', + 'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', + 'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', + 'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', + 'xh', 'yi', 'yo', 'zu'] + +#useful Queue example: https://stackoverflow.com/a/54658363 +jobs = Queue() + +ccenabledl = [] + +recvids = set() +recchans = set() +recmixes = set() +recplayl = set() + +#HSID, SSID, SID cookies required +cookies = loads(open("config.json").read()) + +headers = {"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",} +del cookies + +def prrun(): + while not jobs.empty(): global recvids global recchans global recmixes global recplayl + global ccenabledl + + item = jobs.get() print("Video ID:", str(item).strip()) while True: @@ -29,7 +61,7 @@ class batchthread(threading.Thread): except BaseException as e: print(e) print("Error in retrieving information, waiting 30 seconds") - raise + #raise sleep(30) # Add any discovered videos @@ -46,19 +78,10 @@ class batchthread(threading.Thread): open("out/"+str(item).strip()+"/"+str(item).strip()+"_published_credits.json", "w").write(dumps(info[1])) if info[0]: #ccenabled - while True: - gsres = False - try: - gsres = getsubs(str(item).strip()) - except BaseException as e: - print(e) - if gsres: - break - else: - print("Error in retrieving subtitles, waiting 30 seconds") - sleep(30) + ccenabledl.append(item) + jobs.task_done() - return True + return True # Get a worker ID @@ -81,11 +104,6 @@ while True: except: pass - recvids = set() - recchans = set() - recmixes = set() - recplayl = set() - # Get a batch ID while True: params = ( @@ -96,33 +114,79 @@ while True: if batchrequest.status_code == 200: batchinfo = batchrequest.json() - break - else: - print("Error in retrieving batch assignment, will attempt again in 10 minutes") - sleep(600) + if batchinfo["content"] != "Fail": + break + + print("Error in retrieving batch assignment, will attempt again in 10 minutes") + sleep(600) print("Received batch ID:", batchinfo["batchID"], "Content:", batchinfo["content"]) # Process the batch batchcontent = requests.get(batchinfo["content"]).text.split("\n") - threads = [] while batchcontent: - while len(threads) <= 50 and batchcontent: - item = batchcontent.pop(0) - runthread = batchthread(name = item) - runthread.start() - threads.append(runthread) + jobs.put(batchcontent.pop(0)) + + threads = [] + + for i in range(50): + runthread = Thread(target=prrun) + runthread.start() + threads.append(runthread) + del runthread + + for x in threads: + x.join() + threads.remove(x) + del x - for x in threads: - x.join() - threads.remove(x) + open("out/discoveries.json", "w").write(dumps({"recvids": sorted(recvids), "recchans": sorted(recchans), "recmixes": sorted(recmixes), "recplayl": sorted(recplayl)})) + #clear + recvids.clear() + recchans.clear() + recmixes.clear() + recplayl.clear() + + + subtjobs = Queue() + while ccenabledl: + langcontent = langs.copy() + intvid = ccenabledl.pop(0) + + while langcontent: + subtjobs.put((langcontent.pop(0), intvid)) + del intvid + del langcontent + + subthreads = [] + + for r in range(5): + subrunthread = Thread(target=subprrun, args=(subtjobs,headers)) + subrunthread.start() + subthreads.append(subrunthread) + del subrunthread + + for xa in subthreads: + xa.join() + subthreads.remove(xa) + del xa + # while True: + # gsres = False + # try: + # gsres = getsubs(str(item).strip()) + # except BaseException as e: + # print(e) + # if gsres: + # break + # else: + # print("Error in retrieving subtitles, waiting 30 seconds") + # sleep(30) #https://stackoverflow.com/a/11968881 # TODO: put the data somewhere... # TODO: put the discoveries somewhere... - open("out/discoveries.json", "w").write(dumps({"recvids": sorted(recvids), "recchans": sorted(recchans), "recmixes": sorted(recmixes), "recplayl": sorted(recplayl)})) make_archive("out", "zip", "out") #check this