tech234a
/
ytcc-archive
spegling av https://github.com/Data-Horde/ytcc-archive
1
0
Förgrening 0
Kod Ärenden 0 Släpp 0 Wiki Aktiviteter
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.
103 Incheckningar
7 Grenar
531 KiB
 
 
Träd: 851e7b7241
Grenar Taggar
${ item.name }
Skapa branchen ${ searchTerm }
från '851e7b7241'
${ noResults }
ytcc-archive/worker.py

254 rader
12 KiB
Rå Blame Historik 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import dumps, loads

  7. 

  8. import signal

  9. 

  10. import tracker

  11. 

  12. from youtube_dl import YoutubeDL

  13. 

  14. from shutil import rmtree, which

  15. 

  16. from queue import Queue

  17. 

  18. from gc import collect

  19. 

  20. from discovery import getmetadata

  21. from export import subprrun

  22. 

  23. #useful Queue example: https://stackoverflow.com/a/54658363

  24. jobs = Queue()

  25. 

  26. langcnt = {}

  27. 

  28. try:

  29.     mkdir("out")

  30. except:

  31.     pass

  32. 

  33. try:

  34.     mkdir("directory")

  35. except:

  36.     pass

  37. 

  38. HEROKU = False

  39. if isfile("../Procfile"):

  40.     HEROKU = True

  41. 

  42. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  43.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  44.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  45.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  46.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  47.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  48.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  49.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  50.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  51.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  52.     'xh', 'yi', 'yo', 'zu']

  53. 

  54. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  55. 

  56. #HSID, SSID, SID cookies required

  57. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  58.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  59. elif isfile("config.json"):

  60.     cookies = loads(open("config.json").read())

  61. else:

  62.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  63.     assert False

  64. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  65.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  66.     assert False

  67. 

  68. mysession = requests.session()

  69. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  70. 

  71. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  72. 

  73. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  74. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  75. Language:

  76.   </span>

  77.   English

  78. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  79. 

  80. del validationtest

  81. 

  82. open("cookies.txt", "w").write("""# HTTP Cookie File

  83. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  84. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  85. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  86. 

  87. del cookies

  88. 

  89. #Graceful Shutdown

  90. class GracefulKiller:

  91.     kill_now = False

  92.     def __init__(self):

  93.         signal.signal(signal.SIGINT, self.exit_gracefully)

  94.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  95. 

  96.     def exit_gracefully(self, signum, frame):

  97.         print("Graceful exit process initiated, stopping all tasks...")

  98.         self.kill_now = True

  99. 

  100. gkiller = GracefulKiller()

  101. 

  102. #microtasks

  103. def threadrunner(jobs: Queue):

  104.     global langcnt

  105.     global lasttask

  106.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  107.     while not gkiller.kill_now:

  108.         if not jobs.empty():

  109.             task, vid, args = jobs.get()

  110. 

  111.             if task == "submitdiscovery":

  112.                 tracker.add_item_to_tracker(args, vid)

  113.             elif task == "discovery":

  114.                 while True:

  115.                     try:

  116.                         info = getmetadata(mysession, str(vid).strip())

  117.                         break

  118.                     except BaseException as e:

  119.                         print(e)

  120.                         print("Error in retrieving information, waiting 30 seconds")

  121.                         sleep(30)

  122.                 if info[0] or info[1]: # ccenabled or creditdata

  123.                     if not isdir("out/"+str(vid).strip()):

  124.                         mkdir("out/"+str(vid).strip())

  125.                 if info[1]:

  126.                     open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))

  127. 

  128.                 if info[0]:

  129.                     langcnt[vid] = 0

  130.                     for langcode in langs:

  131.                         jobs.put(("subtitles", vid, langcode))

  132.                 else:

  133.                     jobs.put(("complete", None, "video:"+vid))

  134. 

  135.                 for videodisc in info[2]:

  136.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  137.                 for channeldisc in info[3]:

  138.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  139.                 for mixdisc in info[4]:

  140.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  141.                 for playldisc in info[5]:

  142.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  143. 

  144.             elif task == "subtitles":

  145.                 retval = subprrun(jobs, mysession, args, vid, "default")

  146.                 langcnt[vid] += retval

  147.                 if langcnt[vid] >= 585:

  148.                     jobs.put(("complete", None, "video:"+vid))

  149.             elif task == "subtitles-forceedit-captions":

  150.                 subprrun(jobs, mysession, args, vid, "forceedit-captions")

  151.             elif task == "subtitles-forceedit-metadata":

  152.                 subprrun(jobs, mysession, args, vid, "forceedit-metadata")

  153.             elif task == "channel":

  154.                 while True:

  155.                     try:

  156.                         y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  157.                         for itemyv in y["entries"]:

  158.                             jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  159.                         jobs.put(("complete", None, "channel:"+args))

  160.                         break

  161.                     except:

  162.                         print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  163.             elif task == "playlist":

  164.                 while True:

  165.                     try:

  166.                         y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  167.                         for itemyvp in y["entries"]:

  168.                             jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  169.                         jobs.put(("complete", None, "playlist:"+args))

  170.                         break

  171.                     except:

  172.                         print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  173.             elif task == "complete":

  174.                 size = 0

  175.                 if ":" in args:

  176.                     if args.split(":", 1)[0] == "video":

  177.                         #check if dir is empty, make zip if needed

  178.                         if isdir("out/"+args.split(":", 1)[1]):

  179.                             if not listdir("out/"+args.split(":", 1)[1]):

  180.                                 rmdir("out/"+args.split(":", 1)[1])

  181.                             else:

  182.                                 #zip it up

  183.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  184.                                     mkdir("directory/"+args.split(":", 1)[1])

  185. 

  186.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  187.                                     print("Attempting to zip item...")

  188.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  189. 

  190.                                 #get a target

  191.                                 targetloc = None

  192.                                 while not targetloc:

  193.                                     targetloc = tracker.request_upload_target()

  194.                                     if targetloc:

  195.                                         break

  196.                                     else:

  197.                                         print("Waiting 5 minutes...")

  198.                                         sleep(300)

  199. 

  200.                                 if targetloc.startswith("rsync"):

  201.                                     system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  202.                                 elif targetloc.startswith("http"):

  203.                                     system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  204. 

  205. 

  206.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  207.                                 #cleanup

  208.                                 try:

  209.                                     del langcnt[args.split(":", 1)[1]]

  210.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  211.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  212.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  213.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  214.                                 except:

  215.                                     pass

  216.                 tracker.mark_item_as_done(args, size)

  217.             jobs.task_done()

  218.         else:

  219.             # get a new task from tracker

  220.             collect() #cleanup

  221.             desit = tracker.request_item_from_tracker()

  222.             print("New task:", desit)

  223.             if desit:

  224.                 if desit.split(":", 1)[0] == "video":

  225.                     jobs.put(("discovery", desit.split(":", 1)[1], None))

  226.                 elif desit.split(":", 1)[0] == "channel":

  227.                     jobs.put(("channel", None, desit.split(":", 1)[1]))

  228.                 elif desit.split(":", 1)[0] == "playlist":

  229.                     jobs.put(("playlist", None, desit.split(":", 1)[1]))

  230.                 else:

  231.                     print("Ignoring item for now", desit)

  232.             else:

  233.                 print("Ignoring item for now", desit)

  234.     

  235. 

  236. threads = []

  237. 

  238. THREADCNT = 50

  239. if HEROKU:

  240.     THREADCNT = 20

  241. #now create the rest of the threads

  242. for i in range(THREADCNT):

  243.     runthread = Thread(target=threadrunner, args=(jobs,))

  244.     runthread.start()

  245.     threads.append(runthread)

  246.     del runthread

  247. 

  248. #https://stackoverflow.com/a/11968881

  249. for x in threads:

  250.     x.join()

  251.     threads.remove(x)

  252.     del x

  253. 

  254. print("Exiting...")