tech234a
/
ytcc-archive
mirror of https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
129 Commits
7 Branches
198 KiB
 
 
Branch: http3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'http3'
${ noResults }
ytcc-archive/worker.py

289 lines
18 KiB
Raw Permalink Blame History 

  1. from os import system

  2. from os.path import isfile

  3. from sys import exit

  4. from importlib.util import find_spec

  5. 

  6. HEROKU = False

  7. if isfile("../Procfile") and isfile("../requirements.txt"):

  8.     print("Heroku detected... using 20 threads instead of 50.")

  9.     HEROKU = True

  10. 

  11. if HEROKU:

  12.     if find_spec("aioquic") is None:

  13.         print("Installing aioquic on this Heroku instance since it wasn't installed on deploy...")

  14.         system("python3 -m pip install --user aioquic")

  15.         system("python3 worker.py")

  16.         exit(0)

  17. 

  18. from threading import Thread

  19. import requests

  20. from time import sleep

  21. from os import mkdir, rmdir, listdir, environ

  22. from os.path import isdir, getsize

  23. from json import dumps, loads

  24. 

  25. import signal

  26. 

  27. import tracker

  28. 

  29. from youtube_dl import YoutubeDL

  30. 

  31. from shutil import rmtree, which

  32. 

  33. from queue import Queue

  34. 

  35. from gc import collect

  36. 

  37. from discovery import getmetadata

  38. from export import subprrun

  39. 

  40. #useful Queue example: https://stackoverflow.com/a/54658363

  41. jobs = Queue()

  42. 

  43. try:

  44.     mkdir("out")

  45. except:

  46.     pass

  47. 

  48. try:

  49.     mkdir("directory")

  50. except:

  51.     pass

  52. 

  53. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  54.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  55.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  56.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  57.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  58.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  59.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  60.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  61.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  62.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  63.     'xh', 'yi', 'yo', 'zu']

  64. 

  65. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  66. 

  67. #HSID, SSID, SID cookies required

  68. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  69.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  70. elif isfile("config.json"):

  71.     cookies = loads(open("config.json").read())

  72. else:

  73.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  74.     assert False

  75. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  76.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  77.     assert False

  78. 

  79. mysession = requests.session()

  80. allheaders = {"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",}

  81. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  82. 

  83. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  84. 

  85. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  86. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  87. Language:

  88.   </span>

  89.   English

  90. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  91. 

  92. del validationtest

  93. 

  94. open("cookies.txt", "w").write("""# HTTP Cookie File

  95. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  96. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  97. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  98. 

  99. del cookies

  100. 

  101. #Graceful Shutdown

  102. class GracefulKiller:

  103.     kill_now = False

  104.     def __init__(self):

  105.         signal.signal(signal.SIGINT, self.exit_gracefully)

  106.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  107. 

  108.     def exit_gracefully(self, signum, frame):

  109.         print("Graceful exit process initiated, no longer accepting new tasks but finishing existing ones...")

  110.         self.kill_now = True

  111. 

  112. gkiller = GracefulKiller()

  113. 

  114. #microtasks

  115. def threadrunner():

  116.     jobs = Queue()

  117.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  118.     while True:

  119.         if not jobs.empty():

  120.             task, vid, args = jobs.get()

  121.             if task == "submitdiscovery":

  122.                 tracker.add_item_to_tracker(args, vid)

  123.             elif task == "discovery":

  124.                 while True:

  125.                     try:

  126.                         info = getmetadata(mysession, str(vid).strip(), allheaders)

  127.                         break

  128.                     except BaseException as e:

  129.                         print(e)

  130.                         print("Error in retrieving information, waiting 30 seconds and trying again")

  131.                         #raise

  132.                         sleep(30)

  133.                 if info[0] or info[1]: # ccenabled or creditdata

  134.                     if not isdir("out/"+str(vid).strip()):

  135.                         mkdir("out/"+str(vid).strip())

  136.                 if info[1]:

  137.                     open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))

  138. 

  139.                 if info[0]:

  140.                     for langcode in langs:

  141.                         jobs.put(("subtitles", vid, langcode))

  142. 

  143.                     for langcode in langs:

  144.                         jobs.put(("subtitles-forceedit-metadata", vid, langcode))

  145. 

  146.                     for langcode in langs:

  147.                         jobs.put(("subtitles-forceedit-captions", vid, langcode))

  148. 

  149.                 jobs.put(("complete", None, "video:"+vid))

  150. 

  151.                 for videodisc in info[2]:

  152.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  153.                 for channeldisc in info[3]:

  154.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  155.                 for mixdisc in info[4]:

  156.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  157.                 for playldisc in info[5]:

  158.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  159. 

  160.             elif task == "subtitles":

  161.                 subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions, allheaders)

  162.             elif task == "subtitles-forceedit-captions":

  163.                 subprrun(mysession, args, vid, "forceedit-captions", needforcemetadata, needforcecaptions, allheaders)

  164.             elif task == "subtitles-forceedit-metadata":

  165.                 subprrun(mysession, args, vid, "forceedit-metadata", needforcemetadata, needforcecaptions, allheaders)

  166.             elif task == "channel":

  167.                 try:

  168.                     y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  169.                     for itemyv in y["entries"]:

  170.                         jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  171.                     jobs.put(("complete", None, "channel:"+args))

  172.                 except:

  173.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  174.             elif task == "playlist":

  175.                 try:

  176.                     y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  177.                     for itemyvp in y["entries"]:

  178.                         jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  179.                     jobs.put(("complete", None, "playlist:"+args))

  180.                 except:

  181.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  182.             elif task == "complete":

  183.                 size = 0

  184.                 if ":" in args:

  185.                     if args.split(":", 1)[0] == "video":

  186.                         #check if dir is empty, make zip if needed

  187.                         if isdir("out/"+args.split(":", 1)[1]):

  188.                             if not listdir("out/"+args.split(":", 1)[1]):

  189.                                 rmdir("out/"+args.split(":", 1)[1])

  190.                             else:

  191.                                 #zip it up

  192.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  193.                                     mkdir("directory/"+args.split(":", 1)[1])

  194. 

  195.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  196.                                     print("Attempting to zip item...")

  197.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  198. 

  199.                                 #get a target

  200.                                 targetloc = None

  201.                                 while not targetloc:

  202.                                     targetloc = tracker.request_upload_target()

  203.                                     if targetloc:

  204.                                         break

  205.                                     else:

  206.                                         print("Waiting 5 minutes...")

  207.                                         sleep(300)

  208. 

  209.                                 if targetloc.startswith("rsync"):

  210.                                     system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  211.                                 elif targetloc.startswith("http"):

  212.                                     system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  213. 

  214. 

  215.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  216.                                 #cleanup

  217.                                 try:

  218.                                     del langcnt[args.split(":", 1)[1]]

  219.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  220.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  221.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  222.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  223.                                 except:

  224.                                     pass

  225.                 tracker.mark_item_as_done(args, size)

  226.             jobs.task_done()

  227.         else:

  228.             if not gkiller.kill_now:

  229.                 # get a new task from tracker

  230.                 collect() #cleanup

  231.                 desit = tracker.request_item_from_tracker()

  232.                 print("New task:", desit)

  233. 

  234.                 if desit:

  235.                     if desit.split(":", 1)[0] == "video":

  236.                         needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  237.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  238.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  239.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  240.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  241.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  242.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  243.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  244.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  245.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  246.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  247.                         needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  248.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  249.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  250.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  251.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  252.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  253.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  254.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  255.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  256.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  257.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  258.                         jobs.put(("discovery", desit.split(":", 1)[1], None))

  259.                     elif desit.split(":", 1)[0] == "channel":

  260.                         jobs.put(("channel", None, desit.split(":", 1)[1]))

  261.                     elif desit.split(":", 1)[0] == "playlist":

  262.                         jobs.put(("playlist", None, desit.split(":", 1)[1]))

  263.                     else:

  264.                         print("Ignoring item for now", desit)

  265.                 else:

  266.                     print("Ignoring item for now", desit)

  267.             else:

  268.                 break

  269.     

  270. 

  271. threads = []

  272. 

  273. THREADCNT = 50

  274. if HEROKU:

  275.     THREADCNT = 20

  276. #now create the rest of the threads

  277. for i in range(THREADCNT):

  278.     runthread = Thread(target=threadrunner)

  279.     runthread.start()

  280.     threads.append(runthread)

  281.     del runthread

  282. 

  283. #https://stackoverflow.com/a/11968881

  284. for x in threads:

  285.     x.join()

  286.     threads.remove(x)

  287.     del x

  288. 

  289. print("Exiting...")