tech234a
/
ytcc-archive
mirror of https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
144 Commits
7 Branches
162 KiB
 
 
Tree: 80d125ace7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '80d125ace7'
${ noResults }
ytcc-archive/worker.py

312 lines
20 KiB
Raw Blame History 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import loads

  7. 

  8. from youtube_channel import main

  9. 

  10. import signal

  11. 

  12. import tracker

  13. 

  14. from youtube_dl import YoutubeDL

  15. 

  16. from shutil import rmtree, which

  17. 

  18. from queue import Queue

  19. 

  20. from gc import collect

  21. 

  22. from discovery import getmetadata

  23. from export import subprrun

  24. 

  25. #useful Queue example: https://stackoverflow.com/a/54658363

  26. jobs = Queue()

  27. 

  28. try:

  29.     mkdir("out")

  30. except:

  31.     pass

  32. 

  33. try:

  34.     mkdir("directory")

  35. except:

  36.     pass

  37. 

  38. HEROKU = False

  39. if isfile("../Procfile"):

  40.     HEROKU = True

  41. 

  42. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  43.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  44.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  45.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  46.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  47.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  48.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  49.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  50.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  51.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  52.     'xh', 'yi', 'yo', 'zu']

  53. 

  54. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  55. 

  56. #HSID, SSID, SID cookies required

  57. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  58.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  59. elif isfile("config.json"):

  60.     cookies = loads(open("config.json").read())

  61. else:

  62.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  63.     assert False

  64. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  65.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  66.     assert False

  67. 

  68. mysession = requests.session()

  69. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  70. 

  71. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  72. 

  73. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  74. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  75. Language:

  76.   </span>

  77.   English

  78. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  79. 

  80. del validationtest

  81. 

  82. open("cookies.txt", "w").write("""# HTTP Cookie File

  83. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  84. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  85. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  86. 

  87. del cookies

  88. 

  89. validationtimes = 0

  90. 

  91. #Graceful Shutdown

  92. class GracefulKiller:

  93.     kill_now = False

  94.     def __init__(self):

  95.         signal.signal(signal.SIGINT, self.exit_gracefully)

  96.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  97. 

  98.     def exit_gracefully(self, signum, frame):

  99.         print("Graceful exit process initiated, no longer accepting new tasks but finishing existing ones...")

  100.         self.kill_now = True

  101. 

  102. gkiller = GracefulKiller()

  103. 

  104. #microtasks

  105. def threadrunner():

  106.     global validationtimes

  107.     jobs = Queue()

  108.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  109.     while True:

  110.         if not jobs.empty():

  111.             task, vid, args = jobs.get()

  112.             if task == "submitdiscovery":

  113.                 tracker.add_item_to_tracker(args, vid)

  114.             elif task == "discovery":

  115.                 

  116.                 while True:

  117.                     try:

  118.                         info = getmetadata(mysession, str(vid).strip())

  119.                         break

  120.                     except BaseException as e:

  121.                         print(e)

  122.                         print("Error in retrieving information, waiting 30 seconds and trying again")

  123.                         sleep(30)

  124.                 if info[0]: # ccenabled

  125.                     if not isdir("out/"+str(vid).strip()):

  126.                         mkdir("out/"+str(vid).strip())

  127. 

  128.                 if info[0]:

  129.                     for langcode in langs:

  130.                         jobs.put(("subtitles", vid, langcode))

  131. 

  132.                     for langcode in langs:

  133.                         jobs.put(("subtitles-forceedit-metadata", vid, langcode))

  134. 

  135.                     for langcode in langs:

  136.                         jobs.put(("subtitles-forceedit-captions", vid, langcode))

  137. 

  138.                 jobs.put(("complete", None, "video:"+vid))

  139. 

  140.                 for videodisc in info[1]:

  141.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  142.                 for channeldisc in info[2]:

  143.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  144.                 for mixdisc in info[3]:

  145.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  146.                 for playldisc in info[4]:

  147.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  148. 

  149.             elif task == "subtitles":

  150.                 subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions)

  151.             elif task == "subtitles-forceedit-captions":

  152.                 subprrun(mysession, args, vid, "forceedit-captions", needforcemetadata, needforcecaptions)

  153.             elif task == "subtitles-forceedit-metadata":

  154.                 subprrun(mysession, args, vid, "forceedit-metadata", needforcemetadata, needforcecaptions)

  155.             elif task == "channel":

  156.                 try:

  157.                     y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  158.                     for itemyv in y["entries"]:

  159.                         jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  160. 

  161.                     #channel created playlists

  162.                     y = main(desit.split(":", 1)[1])

  163.                     for itemyv in y["playlists"]:

  164.                         jobs.put(("submitdiscovery", itemyv, tracker.ItemType.Playlist))

  165.                     for itemyv in y["channels"]:

  166.                         jobs.put(("submitdiscovery", itemyv, tracker.ItemType.Channel))

  167. 

  168.                     jobs.put(("complete", None, "channel:"+args))

  169.                 except:

  170.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  171.             elif task == "playlist":

  172.                 try:

  173.                     y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  174.                     #TODO: extract owner channel in other projects

  175.                     #TODO: handle channels in other projects, not needed here because we will get it from the video

  176.                     for itemyvp in y["entries"]:

  177.                         jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  178.                     jobs.put(("complete", None, "playlist:"+args))

  179.                 except:

  180.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  181.             elif task == "mixplaylist":

  182.                 try:

  183.                     wptext = mysession.get("https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]).text

  184. 

  185.                     #channel handling not needed here because we will get it from the video

  186.                     for line in wptext.splitlines():

  187.                         if line.strip().startswith('window["ytInitialData"] = '):

  188.                             initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  189.                             for itemyvp in initdata["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"]["contents"]:

  190.                                 jobs.put(("submitdiscovery", itemyvp["playlistPanelVideoRenderer"]["videoId"], tracker.ItemType.Video))

  191. 

  192.                     jobs.put(("complete", None, "mixplaylist:"+args))

  193.                 except:

  194.                     print("Mix Playlist error, ignoring but not marking as complete...", "https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1])

  195.             elif task == "complete":

  196.                 size = 0

  197.                 if ":" in args:

  198.                     if args.split(":", 1)[0] == "video":

  199.                         #check if dir is empty, make zip if needed

  200.                         if isdir("out/"+args.split(":", 1)[1]):

  201.                             if not listdir("out/"+args.split(":", 1)[1]):

  202.                                 rmdir("out/"+args.split(":", 1)[1])

  203.                             else:

  204.                                 #zip it up

  205.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  206.                                     mkdir("directory/"+args.split(":", 1)[1])

  207. 

  208.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  209.                                     print("Attempting to zip item...")

  210.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  211. 

  212.                                 #get a target

  213.                                 targetloc = None

  214.                                 while not targetloc:

  215.                                     targetloc = tracker.request_upload_target()

  216.                                     if targetloc:

  217.                                         break

  218.                                     else:

  219.                                         print("Waiting 5 minutes...")

  220.                                         sleep(300)

  221. 

  222.                                 while True:

  223.                                     if targetloc.startswith("rsync"):

  224.                                         exitinfo = system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  225.                                     elif targetloc.startswith("http"):

  226.                                         exitinfo = system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  227. 

  228.                                     if exitinfo == 0: # note that on Unix this isn't necessarily the exit code but it's still 0 upon successful exit

  229.                                         break

  230.                                     else:

  231.                                         print("Error in sending data to target, waiting 30 seconds and trying again.")

  232.                                         sleep(30)

  233. 

  234. 

  235.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  236.                                 #cleanup

  237.                                 try:

  238.                                     del langcnt[args.split(":", 1)[1]]

  239.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  240.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  241.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  242.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  243.                                 except:

  244.                                     pass

  245.                 tracker.mark_item_as_done(args, size)

  246.             jobs.task_done()

  247.         else:

  248.             if not gkiller.kill_now:

  249.                 # get a new task from tracker

  250.                 collect() #cleanup

  251. 

  252.                 desit = tracker.request_item_from_tracker()

  253.                 print("New task:", desit)

  254. 

  255.                 if desit:

  256.                     if desit.split(":", 1)[0] == "video":

  257.                         needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  258.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  259.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  260.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  261.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  262.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  263.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  264.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  265.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  266.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  267.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  268.                         needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  269.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  270.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  271.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  272.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  273.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  274.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  275.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  276.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  277.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  278.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  279.                         jobs.put(("discovery", desit.split(":", 1)[1], None))

  280.                     elif desit.split(":", 1)[0] == "channel":

  281.                         jobs.put(("channel", None, desit.split(":", 1)[1]))

  282.                     elif desit.split(":", 1)[0] == "playlist":

  283.                         jobs.put(("playlist", None, desit.split(":", 1)[1]))

  284.                     elif desit.split(":", 1)[0] == "mixplaylist":

  285.                         jobs.put(("mixplaylist", None, desit.split(":", 1)[1]))

  286.                     else:

  287.                         print("Ignoring item for now", desit)

  288.                 else:

  289.                     print("Ignoring item for now", desit)

  290.             else:

  291.                 break

  292.     

  293. 

  294. threads = []

  295. 

  296. THREADCNT = 50

  297. if HEROKU:

  298.     THREADCNT = 20

  299. #now create the rest of the threads

  300. for i in range(THREADCNT):

  301.     runthread = Thread(target=threadrunner)

  302.     runthread.start()

  303.     threads.append(runthread)

  304.     del runthread

  305. 

  306. #https://stackoverflow.com/a/11968881

  307. for x in threads:

  308.     x.join()

  309.     threads.remove(x)

  310.     del x

  311. 

  312. print("Exiting...")