tech234a
/
ytcc-archive
의 미러 https://github.com/Data-Horde/ytcc-archive
1
0
포크 0
코드 이슈 0 릴리즈 0 위키 활동
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
25개 이상의 토픽을 선택하실 수 없습니다. Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
122 커밋
7 브랜치
374 KiB
 
 
트리: fd3c0a6dc4
브랜치 태그
${ item.name }
${ searchTerm } 브랜치 생성
from 'fd3c0a6dc4'
${ noResults }
ytcc-archive/worker.py

282 lines
18 KiB
Raw Blame 히스토리 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import dumps, loads

  7. 

  8. HEROKU = False

  9. if isfile("../Procfile") and isfile("../requirements.txt"):

  10.     print("Heroku detected... using 20 threads instead of 50.")

  11.     HEROKU = True

  12. 

  13. if HEROKU:

  14.     if not "aioquic" in open("../requirements.txt").read():

  15.         print("Installing aioquic on this Heroku instance since it wasn't installed on deploy...")

  16.         system("pip install --user aioquic")

  17. 

  18. import signal

  19. 

  20. import tracker

  21. 

  22. from youtube_dl import YoutubeDL

  23. 

  24. from shutil import rmtree, which

  25. 

  26. from queue import Queue

  27. 

  28. from gc import collect

  29. 

  30. from discovery import getmetadata

  31. from export import subprrun

  32. 

  33. #useful Queue example: https://stackoverflow.com/a/54658363

  34. jobs = Queue()

  35. 

  36. try:

  37.     mkdir("out")

  38. except:

  39.     pass

  40. 

  41. try:

  42.     mkdir("directory")

  43. except:

  44.     pass

  45. 

  46. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  47.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  48.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  49.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  50.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  51.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  52.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  53.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  54.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  55.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  56.     'xh', 'yi', 'yo', 'zu']

  57. 

  58. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  59. 

  60. #HSID, SSID, SID cookies required

  61. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  62.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  63. elif isfile("config.json"):

  64.     cookies = loads(open("config.json").read())

  65. else:

  66.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  67.     assert False

  68. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  69.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  70.     assert False

  71. 

  72. mysession = requests.session()

  73. allheaders = {"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",}

  74. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  75. 

  76. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  77. 

  78. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  79. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  80. Language:

  81.   </span>

  82.   English

  83. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  84. 

  85. del validationtest

  86. 

  87. open("cookies.txt", "w").write("""# HTTP Cookie File

  88. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  89. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  90. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  91. 

  92. del cookies

  93. 

  94. #Graceful Shutdown

  95. class GracefulKiller:

  96.     kill_now = False

  97.     def __init__(self):

  98.         signal.signal(signal.SIGINT, self.exit_gracefully)

  99.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  100. 

  101.     def exit_gracefully(self, signum, frame):

  102.         print("Graceful exit process initiated, no longer accepting new tasks but finishing existing ones...")

  103.         self.kill_now = True

  104. 

  105. gkiller = GracefulKiller()

  106. 

  107. #microtasks

  108. def threadrunner():

  109.     jobs = Queue()

  110.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  111.     while True:

  112.         if not jobs.empty():

  113.             task, vid, args = jobs.get()

  114.             if task == "submitdiscovery":

  115.                 tracker.add_item_to_tracker(args, vid)

  116.             elif task == "discovery":

  117.                 while True:

  118.                     try:

  119.                         info = getmetadata(mysession, str(vid).strip(), allheaders)

  120.                         break

  121.                     except BaseException as e:

  122.                         print(e)

  123.                         print("Error in retrieving information, waiting 30 seconds and trying again")

  124.                         #raise

  125.                         sleep(30)

  126.                 if info[0] or info[1]: # ccenabled or creditdata

  127.                     if not isdir("out/"+str(vid).strip()):

  128.                         mkdir("out/"+str(vid).strip())

  129.                 if info[1]:

  130.                     open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))

  131. 

  132.                 if info[0]:

  133.                     for langcode in langs:

  134.                         jobs.put(("subtitles", vid, langcode))

  135. 

  136.                     for langcode in langs:

  137.                         jobs.put(("subtitles-forceedit-metadata", vid, langcode))

  138. 

  139.                     for langcode in langs:

  140.                         jobs.put(("subtitles-forceedit-captions", vid, langcode))

  141. 

  142.                 jobs.put(("complete", None, "video:"+vid))

  143. 

  144.                 for videodisc in info[2]:

  145.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  146.                 for channeldisc in info[3]:

  147.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  148.                 for mixdisc in info[4]:

  149.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  150.                 for playldisc in info[5]:

  151.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  152. 

  153.             elif task == "subtitles":

  154.                 subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions, allheaders)

  155.             elif task == "subtitles-forceedit-captions":

  156.                 subprrun(mysession, args, vid, "forceedit-captions", needforcemetadata, needforcecaptions, allheaders)

  157.             elif task == "subtitles-forceedit-metadata":

  158.                 subprrun(mysession, args, vid, "forceedit-metadata", needforcemetadata, needforcecaptions, allheaders)

  159.             elif task == "channel":

  160.                 try:

  161.                     y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  162.                     for itemyv in y["entries"]:

  163.                         jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  164.                     jobs.put(("complete", None, "channel:"+args))

  165.                 except:

  166.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  167.             elif task == "playlist":

  168.                 try:

  169.                     y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  170.                     for itemyvp in y["entries"]:

  171.                         jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  172.                     jobs.put(("complete", None, "playlist:"+args))

  173.                 except:

  174.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  175.             elif task == "complete":

  176.                 size = 0

  177.                 if ":" in args:

  178.                     if args.split(":", 1)[0] == "video":

  179.                         #check if dir is empty, make zip if needed

  180.                         if isdir("out/"+args.split(":", 1)[1]):

  181.                             if not listdir("out/"+args.split(":", 1)[1]):

  182.                                 rmdir("out/"+args.split(":", 1)[1])

  183.                             else:

  184.                                 #zip it up

  185.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  186.                                     mkdir("directory/"+args.split(":", 1)[1])

  187. 

  188.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  189.                                     print("Attempting to zip item...")

  190.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  191. 

  192.                                 #get a target

  193.                                 targetloc = None

  194.                                 while not targetloc:

  195.                                     targetloc = tracker.request_upload_target()

  196.                                     if targetloc:

  197.                                         break

  198.                                     else:

  199.                                         print("Waiting 5 minutes...")

  200.                                         sleep(300)

  201. 

  202.                                 if targetloc.startswith("rsync"):

  203.                                     system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  204.                                 elif targetloc.startswith("http"):

  205.                                     system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  206. 

  207. 

  208.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  209.                                 #cleanup

  210.                                 try:

  211.                                     del langcnt[args.split(":", 1)[1]]

  212.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  213.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  214.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  215.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  216.                                 except:

  217.                                     pass

  218.                 tracker.mark_item_as_done(args, size)

  219.             jobs.task_done()

  220.         else:

  221.             if not gkiller.kill_now:

  222.                 # get a new task from tracker

  223.                 collect() #cleanup

  224.                 desit = tracker.request_item_from_tracker()

  225.                 print("New task:", desit)

  226. 

  227.                 if desit:

  228.                     if desit.split(":", 1)[0] == "video":

  229.                         needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  230.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  231.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  232.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  233.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  234.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  235.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  236.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  237.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  238.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  239.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  240.                         needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  241.                             'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  242.                             'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  243.                             'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  244.                             'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  245.                             'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  246.                             'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  247.                             'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  248.                             'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  249.                             'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  250.                             'xh': None, 'yi': None, 'yo': None, 'zu': None}

  251.                         jobs.put(("discovery", desit.split(":", 1)[1], None))

  252.                     elif desit.split(":", 1)[0] == "channel":

  253.                         jobs.put(("channel", None, desit.split(":", 1)[1]))

  254.                     elif desit.split(":", 1)[0] == "playlist":

  255.                         jobs.put(("playlist", None, desit.split(":", 1)[1]))

  256.                     else:

  257.                         print("Ignoring item for now", desit)

  258.                 else:

  259.                     print("Ignoring item for now", desit)

  260.             else:

  261.                 break

  262.     

  263. 

  264. threads = []

  265. 

  266. THREADCNT = 50

  267. if HEROKU:

  268.     THREADCNT = 20

  269. #now create the rest of the threads

  270. for i in range(THREADCNT):

  271.     runthread = Thread(target=threadrunner)

  272.     runthread.start()

  273.     threads.append(runthread)

  274.     del runthread

  275. 

  276. #https://stackoverflow.com/a/11968881

  277. for x in threads:

  278.     x.join()

  279.     threads.remove(x)

  280.     del x

  281. 

  282. print("Exiting...")