tech234a
/
ytcc-archive
espelhamento de https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Código Issues 0 Versões 0 Wiki Atividade
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Você não pode selecionar mais de 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
87 Commits
7 Branches
531 KiB
 
 
Tag: a96d4c7b48
Branches Tags
${ item.name }
Criar branch ${ searchTerm }
de a96d4c7b48
${ noResults }
ytcc-archive/worker.py

246 linhas
10 KiB
Original Anotar Histórico 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import dumps, loads

  7. 

  8. import signal

  9. 

  10. import tracker

  11. 

  12. from youtube_dl import YoutubeDL

  13. 

  14. from shutil import make_archive, rmtree

  15. 

  16. from queue import Queue

  17. 

  18. from gc import collect

  19. 

  20. from discovery import getmetadata

  21. from export import subprrun

  22. 

  23. #useful Queue example: https://stackoverflow.com/a/54658363

  24. jobs = Queue()

  25. 

  26. langcnt = {}

  27. 

  28. try:

  29.     mkdir("out")

  30. except:

  31.     pass

  32. 

  33. try:

  34.     mkdir("directory")

  35. except:

  36.     pass

  37. 

  38. HEROKU = False

  39. if isfile("../Procfile"):

  40.     HEROKU = True

  41. 

  42. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  43.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  44.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  45.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  46.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  47.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  48.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  49.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  50.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  51.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  52.     'xh', 'yi', 'yo', 'zu']

  53. 

  54. #HSID, SSID, SID cookies required

  55. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  56.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  57. elif isfile("config.json"):

  58.     cookies = loads(open("config.json").read())

  59. else:

  60.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  61.     assert False

  62. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  63.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  64.     assert False

  65. 

  66. mysession = requests.session()

  67. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  68. 

  69. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  70. 

  71. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  72. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  73. Language:

  74.   </span>

  75.   English

  76. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  77. 

  78. del validationtest

  79. 

  80. open("cookies.txt", "w").write("""# HTTP Cookie File

  81. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  82. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  83. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  84. 

  85. del cookies

  86. 

  87. #Graceful Shutdown

  88. class GracefulKiller:

  89.     kill_now = False

  90.     def __init__(self):

  91.         signal.signal(signal.SIGINT, self.exit_gracefully)

  92.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  93. 

  94.     def exit_gracefully(self,signum, frame):

  95.         self.kill_now = True

  96. 

  97. gkiller = GracefulKiller()

  98. 

  99. #TODO: zipping, completion of subtitles (return value), limit task retrieval count

  100. 

  101. #microtasks

  102. def threadrunner(jobs: Queue):

  103.     global langcnt

  104.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  105.     while not gkiller.kill_now:

  106.         if not jobs.empty():

  107.             task, vid, args = jobs.get()

  108. 

  109.             if task == "submitdiscovery":

  110.                 tracker.add_item_to_tracker(args, vid)

  111.             elif task == "discovery":

  112.                 while True:

  113.                     try:

  114.                         info = getmetadata(mysession, str(vid).strip())

  115.                         break

  116.                     except BaseException as e:

  117.                         print(e)

  118.                         print("Error in retrieving information, waiting 30 seconds")

  119.                         sleep(30)

  120.                 if info[0] or info[1]: # ccenabled or creditdata

  121.                     if not isdir("out/"+str(vid).strip()):

  122.                         mkdir("out/"+str(vid).strip())

  123.                 if info[1]:

  124.                     open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))

  125. 

  126.                 if info[0]:

  127.                     langcnt[vid] = 0

  128.                     for langcode in langs:

  129.                         jobs.put(("subtitles", vid, langcode))

  130.                 else:

  131.                     jobs.put(("complete", None, "video:"+vid))

  132. 

  133.                 for videodisc in info[2]:

  134.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  135.                 for channeldisc in info[3]:

  136.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  137.                 for mixdisc in info[4]:

  138.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  139.                 for playldisc in info[5]:

  140.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  141. 

  142.             elif task == "subtitles":

  143.                 subprrun(jobs, mysession, args, vid, "default")

  144.                 langcnt[vid] += 1

  145.                 if langcnt[vid] >= 195:

  146.                     pass #complete(?)

  147.             elif task == "subtitles-forceedit-captions":

  148.                 subprrun(jobs, mysession, args, vid, "forceedit-captions")

  149.             elif task == "subtitles-forceedit-metadata":

  150.                 subprrun(jobs, mysession, args, vid, "forceedit-metadata")

  151.             elif task == "channel":

  152.                 y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  153.                 for itemyv in y["entries"]:

  154.                     jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  155.                 jobs.put(("complete", None, "channel:"+args))

  156.             elif task == "playlist":

  157.                 y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  158.                 for itemyvp in y["entries"]:

  159.                     jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  160.                 jobs.put(("complete", None, "playlist:"+args))

  161.             elif task == "complete":

  162.                 size = 0

  163.                 if ":" in args:

  164.                     if args.split(":", 1)[0] == "video":

  165.                         #check if dir is empty, make zip if needed

  166.                         if isfile("directory/"+args.split(":", 1)[1]+".zip"):

  167.                             size = getsize("directory/"+args.split(":", 1)[1]+".zip")

  168. 

  169.                             #get a target

  170.                             targetloc = None

  171.                             while not targetloc:

  172.                                 targetloc = tracker.request_upload_target()

  173.                                 if targetloc:

  174.                                     break

  175.                                 else:

  176.                                     print("Waiting 5 minutes...")

  177.                                     sleep(300)

  178. 

  179.                             if targetloc.startswith("rsync"):

  180.                                 system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- directory/ "+targetloc)

  181.                             elif targetloc.startswith("http"):

  182.                                 for filzip in listdir("directory"):

  183.                                     if filzip.endswith(".zip"):

  184.                                         system("curl -F "+filzip+"=@directory/"+filzip+" "+targetloc)

  185.                                         tracker.mark_item_as_done(args, size)

  186. 

  187.             jobs.task_done()

  188.         else:

  189.             # get a new task from tracker

  190.             collect() #cleanup

  191.             desit = tracker.request_item_from_tracker()

  192.             if desit:

  193.                 if desit.split(":", 1)[0] == "video":

  194.                     jobs.put(("discovery", desit.split(":", 1)[1], None))

  195.                 elif desit.split(":", 1)[0] == "channel":

  196.                     jobs.put(("channel", None, desit.split(":", 1)[1]))

  197.                 elif desit.split(":", 1)[0] == "playlist":

  198.                     jobs.put(("playlist", None, desit.split(":", 1)[1]))

  199.                 else:

  200.                     print("Ignoring item for now", desit)

  201.             else:

  202.                 print("Ignoring item for now", desit)

  203. 

  204. def prrun():

  205. 

  206.         print("Video ID:", str(item).strip())

  207.         

  208. 

  209. while not gkiller.kill_now:

  210.     

  211. 

  212.     threads = []

  213. 

  214.     for i in range(50):

  215.         runthread = Thread(target=prrun)

  216.         runthread.start()

  217.         threads.append(runthread)

  218.         del runthread

  219. 

  220.     for x in threads:

  221.         x.join()

  222.         threads.remove(x)

  223.         del x

  224. 

  225.     for fol in listdir("out"): #remove empty folders

  226.         try:

  227.             if isdir("out/"+fol):

  228.                 rmdir("out/"+fol)

  229.         except:

  230.             pass

  231. 

  232.     #https://stackoverflow.com/a/11968881

  233. 

  234. 

  235.     for fol in listdir("out"):

  236.         if isdir("out/"+fol):

  237.             make_archive("directory/"+fol, "zip", "out/"+fol)

  238. 

  239. 

  240. 

  241. 

  242. 

  243. 

  244.     # clear the output directories

  245.     rmtree("out")

  246.     rmtree("directory")