tech234a
/
ytcc-archive
mirror of https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
134 Commits
7 Branches
294 KiB
 
 
Tree: e07ec3b3da
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e07ec3b3da'
${ noResults }
ytcc-archive/worker.py

316 lines
20 KiB
Raw Blame History 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import dumps, loads

  7. 

  8. import signal

  9. 

  10. import tracker

  11. 

  12. from youtube_dl import YoutubeDL

  13. 

  14. from shutil import rmtree, which

  15. 

  16. from queue import Queue

  17. 

  18. from gc import collect

  19. 

  20. from discovery import getmetadata

  21. from export import subprrun

  22. 

  23. #useful Queue example: https://stackoverflow.com/a/54658363

  24. jobs = Queue()

  25. 

  26. try:

  27.     mkdir("out")

  28. except:

  29.     pass

  30. 

  31. try:

  32.     mkdir("directory")

  33. except:

  34.     pass

  35. 

  36. HEROKU = False

  37. if isfile("../Procfile"):

  38.     HEROKU = True

  39. 

  40. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  41.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  42.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  43.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  44.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  45.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  46.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  47.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  48.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  49.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  50.     'xh', 'yi', 'yo', 'zu']

  51. 

  52. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  53. 

  54. #HSID, SSID, SID cookies required

  55. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  56.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  57. elif isfile("config.json"):

  58.     cookies = loads(open("config.json").read())

  59. else:

  60.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  61.     assert False

  62. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  63.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  64.     assert False

  65. 

  66. mysession = requests.session()

  67. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  68. 

  69. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  70. 

  71. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  72. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  73. Language:

  74.   </span>

  75.   English

  76. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  77. 

  78. del validationtest

  79. 

  80. open("cookies.txt", "w").write("""# HTTP Cookie File

  81. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  82. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  83. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  84. 

  85. del cookies

  86. 

  87. validationtimes = 0

  88. shouldgetjob = True

  89. 

  90. #Graceful Shutdown

  91. class GracefulKiller:

  92.     kill_now = False

  93.     def __init__(self):

  94.         signal.signal(signal.SIGINT, self.exit_gracefully)

  95.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  96. 

  97.     def exit_gracefully(self, signum, frame):

  98.         print("Graceful exit process initiated, no longer accepting new tasks but finishing existing ones...")

  99.         self.kill_now = True

  100. 

  101. gkiller = GracefulKiller()

  102. 

  103. #REMOVED PANIC MECHANISM!

  104. """

  105. enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0]

  106. if not enres:

  107.     print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  108.     shouldgetjob = False

  109.     gkiller.kill_now = True #exit the script

  110. 

  111. del enres

  112. """

  113. 

  114. #microtasks

  115. def threadrunner():

  116.     global shouldgetjob

  117.     global validationtimes

  118.     jobs = Queue()

  119.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  120.     while True:

  121.         if not jobs.empty():

  122.             task, vid, args = jobs.get()

  123.             if task == "submitdiscovery":

  124.                 tracker.add_item_to_tracker(args, vid)

  125.                 #pass

  126.             elif task == "discovery":

  127.                 

  128.                 while True:

  129.                     try:

  130.                         info = getmetadata(mysession, str(vid).strip())

  131.                         break

  132.                     except BaseException as e:

  133.                         print(e)

  134.                         print("Error in retrieving information, waiting 30 seconds and trying again")

  135.                         sleep(30)

  136.                 if info[0] or info[1]: # ccenabled or creditdata

  137.                     if not isdir("out/"+str(vid).strip()):

  138.                         mkdir("out/"+str(vid).strip())

  139.                 if info[1]:

  140.                     open("out/"+str(vid).strip()+"/"+str(vid).strip()+"_published_credits.json", "w").write(dumps(info[1]))

  141. 

  142.                 if info[0]:

  143.                     for langcode in langs:

  144.                         jobs.put(("subtitles", vid, langcode))

  145. 

  146.                     for langcode in langs:

  147.                         jobs.put(("subtitles-forceedit-metadata", vid, langcode))

  148. 

  149.                     for langcode in langs:

  150.                         jobs.put(("subtitles-forceedit-captions", vid, langcode))

  151. 

  152.                 jobs.put(("complete", None, "video:"+vid))

  153. 

  154.                 for videodisc in info[2]:

  155.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  156.                 for channeldisc in info[3]:

  157.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  158.                 for mixdisc in info[4]:

  159.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  160.                 for playldisc in info[5]:

  161.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  162.                 

  163.                 jobs.put(("complete", None, "video:"+vid))

  164.                 #pass

  165. 

  166.             elif task == "subtitles":

  167.                 subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions)

  168.             elif task == "subtitles-forceedit-captions":

  169.                 subprrun(mysession, args, vid, "forceedit-captions", needforcemetadata, needforcecaptions)

  170.             elif task == "subtitles-forceedit-metadata":

  171.                 subprrun(mysession, args, vid, "forceedit-metadata", needforcemetadata, needforcecaptions)

  172.             elif task == "channel":

  173.                 try:

  174.                     y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  175.                     for itemyv in y["entries"]:

  176.                         jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  177.                     jobs.put(("complete", None, "channel:"+args))

  178.                 except:

  179.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  180.             elif task == "playlist":

  181.                 try:

  182.                     y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  183.                     for itemyvp in y["entries"]:

  184.                         jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  185.                     jobs.put(("complete", None, "playlist:"+args))

  186.                 except:

  187.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  188.             elif task == "complete":

  189.                 size = 0

  190.                 if ":" in args:

  191.                     if args.split(":", 1)[0] == "video":

  192.                         #check if dir is empty, make zip if needed

  193.                         if isdir("out/"+args.split(":", 1)[1]):

  194.                             if not listdir("out/"+args.split(":", 1)[1]):

  195.                                 rmdir("out/"+args.split(":", 1)[1])

  196.                             else:

  197.                                 #zip it up

  198.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  199.                                     mkdir("directory/"+args.split(":", 1)[1])

  200. 

  201.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  202.                                     print("Attempting to zip item...")

  203.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  204. 

  205.                                 #get a target

  206.                                 targetloc = None

  207.                                 while not targetloc:

  208.                                     targetloc = tracker.request_upload_target()

  209.                                     if targetloc:

  210.                                         break

  211.                                     else:

  212.                                         print("Waiting 5 minutes...")

  213.                                         sleep(300)

  214. 

  215.                                 if targetloc.startswith("rsync"):

  216.                                     system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  217.                                 elif targetloc.startswith("http"):

  218.                                     system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  219. 

  220. 

  221.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  222.                                 #cleanup

  223.                                 try:

  224.                                     del langcnt[args.split(":", 1)[1]]

  225.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  226.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  227.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  228.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  229.                                 except:

  230.                                     pass

  231.                 tracker.mark_item_as_done(args, size)

  232.             jobs.task_done()

  233.         else:

  234.             if not gkiller.kill_now:

  235.                 # get a new task from tracker

  236.                 collect() #cleanup

  237. 

  238.                 #Protection Mechanism Disarmed

  239.                 """

  240.                 #check that the account has community contributions enabled every 50th item

  241.                 validationtimes += 1

  242.                 if not validationtimes % 50:

  243.                     enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0]

  244.                     if not enres:

  245.                         print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  246.                         shouldgetjob = False

  247.                         gkiller.kill_now = True #exit the script

  248.                     del enres

  249.                 """

  250. 

  251.                 if shouldgetjob:

  252.                     desit = tracker.request_item_from_tracker()

  253.                     print("New task:", desit)

  254. 

  255.                     if desit:

  256.                         if desit.split(":", 1)[0] == "video":

  257.                             needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  258.                                 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  259.                                 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  260.                                 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  261.                                 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  262.                                 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  263.                                 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  264.                                 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  265.                                 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  266.                                 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  267.                                 'xh': None, 'yi': None, 'yo': None, 'zu': None}

  268.                             needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  269.                                 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  270.                                 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  271.                                 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  272.                                 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  273.                                 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  274.                                 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  275.                                 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  276.                                 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  277.                                 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  278.                                 'xh': None, 'yi': None, 'yo': None, 'zu': None}

  279.                             jobs.put(("discovery", desit.split(":", 1)[1], None))

  280.                         elif desit.split(":", 1)[0] == "channel":

  281.                             jobs.put(("channel", None, desit.split(":", 1)[1]))

  282.                         elif desit.split(":", 1)[0] == "playlist":

  283.                             jobs.put(("playlist", None, desit.split(":", 1)[1]))

  284.                         else:

  285.                             print("Ignoring item for now", desit)

  286.                     else:

  287.                         print("Ignoring item for now", desit)

  288.                 else:

  289.                     break

  290.             else:

  291.                 break

  292.     

  293. 

  294. threads = []

  295. 

  296. THREADCNT = 50

  297. if HEROKU:

  298.     THREADCNT = 20

  299. #now create the rest of the threads

  300. for i in range(THREADCNT):

  301.     runthread = Thread(target=threadrunner)

  302.     runthread.start()

  303.     threads.append(runthread)

  304.     del runthread

  305. 

  306. #https://stackoverflow.com/a/11968881

  307. for x in threads:

  308.     x.join()

  309.     threads.remove(x)

  310.     del x

  311. 

  312. if not shouldgetjob:

  313.     print("PROTECTION MECHANISM #3 WAS SOMEHOW TRIGERRED")

  314.     print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  315. 

  316. print("Exiting...")