tech234a
/
ytcc-archive
Mirror von https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Aktivität
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.
125 Commits
7 Branches
162 KiB
 
 
Struktur: 20752675ce
Branches Tags
${ item.name }
Erstelle Branch ${ searchTerm }
von „20752675ce“
${ noResults }
ytcc-archive/discovery.py

163 Zeilen
14 KiB
Originalformat Blame Verlauf 

  1. from time import sleep

  2. from typing import Dict

  3. from json import loads

  4. 

  5. langcodes = {"Afar": "aa", "Abkhazian": "ab", "Afrikaans": "af", "Akan": "ak", "all": "all", "Amharic": "am", "Aragonese": "an", "Arabic": "ar", "Aramaic": "arc", "Algerian Arabic": "arq", "Assamese": "as", "American Sign Language": "ase", "Asturian": "ast", "Avaric": "av", "Aymara": "ay", "Azerbaijani": "az", "Bashkir": "ba", "Belarusian": "be", "Bulgarian": "bg", "Bihari": "bh", "Bislama": "bi", "Bangla": "bn", "Tibetan": "bo", "Breton": "br", "Bosnian": "bs", "Catalan": "ca", "Cebuano": "ceb", "Choctaw": "cho", "Cherokee": "chr", "Corsican": "co", "Czech": "cs", "Church Slavic": "cu", "Welsh": "cy", "Danish": "da", "Danish (Denmark)": "da-DK", "German": "de", "German (Austria)": "de-AT", "German (Switzerland)": "de-CH", "German (Germany)": "de-DE", "Divehi": "dv", "Dzongkha": "dz", "Ewe": "ee", "Greek": "el", "English": "en", "English (United Arab Emirates)": "en-AE", "English (Canada)": "en-CA", "English (United Kingdom)": "en-GB", "English (Ireland)": "en-IE", "English (India)": "en-IN", "English (United States)": "en-US", "Esperanto": "eo", "Spanish": "es", "Spanish (Latin America)": "es-419", "Spanish (Argentina)": "es-AR", "Spanish (Chile)": "es-CL", "Spanish (Colombia)": "es-CO", "Spanish (Costa Rica)": "es-CR", "Spanish (Spain)": "es-ES", "Spanish (Mexico)": "es-MX", "Spanish (Nicaragua)": "es-NI", "Spanish (United States)": "es-US", "Estonian": "et", "Basque": "eu", "Persian": "fa", "Persian (Afghanistan)": "fa-AF", "Persian (Iran)": "fa-IR", "Fulah": "ff", "Finnish": "fi", "Filipino": "fil", "Fijian": "fj", "Faroese": "fo", "French": "fr", "French (Belgium)": "fr-BE", "French (Canada)": "fr-CA", "French (Switzerland)": "fr-CH", "French (France)": "fr-FR", "Western Frisian": "fy", "Irish": "ga", "Scottish Gaelic": "gd", "Galician": "gl", "Guarani": "gn", "Swiss German": "gsw", "Gujarati": "gu", "Hausa": "ha", "Hakka Chinese": "hak", "Hakka Chinese (Taiwan)": "hak-TW", "Hindi": "hi-Latn", "Hmong": "hmn", "Croatian": "hr", "Haitian Creole": "ht", "Hungarian": "hu", "Armenian": "hy", "Interlingua": "ia", "Indonesian": "id", "Interlingue": "ie", "Igbo": "ig", "Sichuan Yi": "ii", "Inupiaq": "ik", "Icelandic": "is", "Italian": "it", "Italian (Italy)": "it-IT", "Inuktitut": "iu", "Hebrew": "iw", "Japanese": "ja", "Javanese": "jv", "Georgian": "ka", "Kazakh": "kk", "Kalaallisut": "kl", "Khmer": "km", "Kannada": "kn", "Korean": "ko", "Korean (South Korea)": "ko-KR", "Kanuri": "kr", "Kashmiri": "ks", "Kurdish": "ku", "Kyrgyz": "ky", "Latin": "la", "Luxembourgish": "lb", "Lingala": "ln", "Lao": "lo", "Lithuanian": "lt", "Mizo": "lus", "Latvian": "lv", "Masai": "mas", "Malagasy": "mg", "Maori": "mi", "Miscellaneous languages": "mis", "Macedonian": "mk", "Malayalam": "ml", "Mongolian": "mn", "Manipuri": "mni", "Moldavian": "mo", "Marathi": "mr", "Malay": "ms", "Maltese": "mt", "Burmese": "my", "Nauru": "na", "Min Nan Chinese": "nan", "Min Nan Chinese (Taiwan)": "nan-TW", "Nepali": "ne", "Dutch": "nl", "Dutch (Belgium)": "nl-BE", "Dutch (Netherlands)": "nl-NL", "Norwegian Nynorsk": "nn", "Norwegian": "no", "not": "not", "Navajo": "nv", "Occitan": "oc", "Oromo": "om", "Odia": "or", "Punjabi": "pa", "Polish": "pl", "Polish (Poland)": "pl-PL", "Pashto": "ps", "Portuguese": "pt", "Portuguese (Brazil)": "pt-BR", "Portuguese (Portugal)": "pt-PT", "Quechua": "qu", "Romansh": "rm", "Rundi": "rn", "Romanian": "ro", "Romanian (Moldova)": "ro-MD", "Russian": "ru-Latn", "Russian (Russia)": "ru-RU", "Kinyarwanda": "rw", "Sanskrit": "sa", "Sardinian": "sc", "Sicilian": "scn", "Scots": "sco", "Sindhi": "sd", "Sherdukpen": "sdp", "Northern Sami": "se", "Sango": "sg", "Serbo-Croatian": "sh", "Sinhala": "si", "Slovak": "sk", "Slovenian": "sl", "Samoan": "sm", "Shona": "sn", "Somali": "so", "Albanian": "sq", "Serbian": "sr", "Serbian (Cyrillic)": "sr-Cyrl", "Serbian (Latin)": "sr-Latn", "Swati": "ss", "Southern Sotho": "st", "Sundanese": "su", "Swedish": "sv", "Swahili": "sw", "Tamil": "ta", "Telugu": "te", "Tajik": "tg", "Thai": "th", "Tigrinya": "ti", "Turkmen": "tk", "Tagalog": "tl", "Klingon": "tlh", "Tswana": "tn", "Tongan": "to", "Turkish": "tr", "Turkish (Turkey)": "tr-TR", "Tsonga": "ts", "Tatar": "tt", "Twi": "tw", "Ukrainian": "uk", "Urdu": "ur", "Uzbek": "uz", "Vietnamese": "vi", "Volap\\xFCk": "vo", "Wolof": "wo", "Xhosa": "xh", "Yiddish": "yi", "Yoruba": "yo", "Cantonese": "yue", "Cantonese (Hong Kong)": "yue-HK", "Chinese": "zh", "Chinese (China)": "zh-CN", "Chinese (Hong Kong)": "zh-HK", "Chinese (Simplified)": "zh-Hans", "Chinese (Simplified, China)": "zh-Hans-CN", "Chinese (Simplified, Singapore)": "zh-Hans-SG", "Chinese (Traditional)": "zh-Hant", "Chinese (Traditional, Hong Kong)": "zh-Hant-HK", "Chinese (Traditional, Taiwan)": "zh-Hant-TW", "Chinese (Singapore)": "zh-SG", "Chinese (Taiwan)": "zh-TW", "Zulu": "zu", "Hiri Motu": "ho", "Tok Pisin": "tpi", "Voro": "vor"}

  6. 

  7. def getmetadata(mysession, vid, ccenabledonly=False):

  8.     params = (

  9.         ("v", vid),

  10.     )

  11. 

  12.     while True:

  13.         wpage = mysession.get("https://www.youtube.com/watch", params=params)

  14.         if not """</div><div id="content" class="  content-alignment" role="main"><p class='largeText'>Sorry for the interruption. We have been receiving a large volume of requests from your network.</p>

  15. 

  16. <p>To continue with your YouTube experience, please fill out the form below.</p>""" in wpage.text and not wpage.status_code == 429 and 'window["ytInitialPlayerResponse"] = ' in wpage.text and 'window["ytInitialData"] = ' in wpage.text:

  17.             break

  18.         else:

  19.             print("Captcha detected, waiting 30 seconds")

  20.             sleep(30)

  21. 

  22.     wptext = wpage.text

  23. 

  24.     initplay = None

  25.     initdata = None

  26. 

  27.     recvids  = set()

  28.     recchans = set()

  29.     recmixes = set()

  30.     recplayl = set()

  31. 

  32.     ccenabled = False #default values

  33.     creditdata = {}

  34. 

  35.     for line in wptext.splitlines():

  36.         if line.strip().startswith('window["ytInitialPlayerResponse"] = '):

  37.             initplay = loads(line.split('window["ytInitialPlayerResponse"] = ', 1)[1].strip()[:-1])

  38. 

  39.             if not ccenabledonly:

  40.                 if initplay["playabilityStatus"]["status"] == "ERROR":

  41.                     print(vid, "unavailable")

  42.                     return False, {}, recvids, recchans, recmixes, recplayl

  43.                 

  44.                 if "endscreen" in initplay.keys():

  45.                     if "endscreenRenderer" in initplay["endscreen"].keys():

  46.                         for el in initplay["endscreen"]["endscreenRenderer"]:

  47. 

  48.                             if type(el) == Dict:

  49.                                 elint = el["endscreenElementRenderer"]

  50. 

  51.                                 if "endscreenElementRenderer" in el.keys():

  52.                                     if elint["style"] == "VIDEO":

  53.                                         recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  54. 

  55.                                     elif elint["style"] == "CHANNEL":

  56.                                         try:

  57.                                             recchans.add(elint["endpoint"]["browseEndpoint"]["browseId"])

  58.                                         except:

  59.                                             print("Channel endscreen error")

  60.                                             raise

  61. 

  62.                                     elif elint["style"] == "PLAYLIST":

  63.                                         recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  64.                                         recplayl.add(elint["endpoint"]["watchEndpint"]["playlistId"])

  65. 

  66.             if "captions" in initplay.keys():

  67.                 ccenabled = "contribute" in initplay["captions"]["playerCaptionsRenderer"]

  68.             else:

  69.                 ccenabled = False # if captions information is not present, community contributions are not enabled

  70. 

  71.             if not ccenabledonly:

  72.                 if "videoDetails" in initplay.keys():

  73.                     if "channelId" in initplay["videoDetails"].keys():

  74.                         recchans.add(initplay["videoDetails"]["channelId"])

  75.         elif line.strip().startswith('window["ytInitialData"] = '):

  76.             if not ccenabledonly:

  77.                 initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  78.                 if "contents" in initdata.keys(): #prevent exception

  79.                     try:

  80.                         if "results" in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"].keys():

  81.                             for recmd in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"]["results"]:

  82.                                 #auto is like the others

  83.                                 if "compactAutoplayRenderer" in recmd.keys():

  84.                                     recmd = recmd["compactAutoplayRenderer"]["contents"][0]

  85. 

  86.                                 if "compactVideoRenderer" in recmd.keys():

  87.                                     recvids.add(recmd["compactVideoRenderer"]["videoId"])

  88.                                     try:

  89.                                         recchans.add(recmd["compactVideoRenderer"]["channelId"])

  90.                                     except KeyError as e:

  91.                                         try:

  92.                                             recchans.add(recmd["compactVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  93.                                         except KeyError as e:

  94.                                             print("Channel extract error")

  95.                                         #raise

  96.                                         #print("Unable to extract channel:")

  97.                                         #print(recmd["compactVideoRenderer"])

  98. 

  99.                                 elif "compactPlaylistRenderer" in recmd.keys():

  100.                                     recplayl.add(recmd["compactPlaylistRenderer"]["playlistId"])

  101.                                     if "navigationEndpoint" in recmd["compactPlaylistRenderer"].keys():

  102.                                         recvids.add(recmd["compactPlaylistRenderer"]["navigationEndpoint"]["watchEndpoint"]["videoId"])

  103.                                     if "navigationEndpoint" in recmd["compactPlaylistRenderer"]["shortBylineText"].keys():

  104.                                         recchans.add(recmd["compactPlaylistRenderer"]["shortBylineText"]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  105. 

  106.                                 elif "compactRadioRenderer" in recmd.keys(): #mix playlist

  107.                                     recmixes.add(recmd["compactRadioRenderer"]["playlistId"])

  108.                                 # todo: find out if channels can be suggested

  109.                     except BaseException as e:

  110.                         print(e)

  111.                         print("Exception in discovery, continuing anyway")

  112. 

  113.             creditdata = {}

  114. 

  115.             if not ccenabledonly:

  116.                 try:

  117.                     mdinfo = initdata["contents"]["twoColumnWatchNextResults"]["results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"]["metadataRowContainer"]["metadataRowContainerRenderer"]["rows"]

  118.                     for item in mdinfo:

  119.                         if item["metadataRowRenderer"]["title"]["simpleText"].startswith("Caption author"): #the request to /watch needs to be in English for this to work

  120.                             try:

  121.                                 desl = langcodes[item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]]

  122.                             except KeyError as e:

  123.                                 #print(e)

  124.                                 print("Language code conversion error, using language name")

  125.                                 desl = item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]

  126.                             creditdata[desl] = []

  127.                             for itemint in item["metadataRowRenderer"]["contents"]:

  128.                                 creditdata[desl].append({"name": itemint["runs"][0]["text"], "channel": itemint["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]})

  129. 

  130.                 except KeyError as e:

  131.                     #print("Video does not have credits")

  132.                     pass

  133.                     #raise

  134.                     #print(e)

  135.         

  136.         if initplay and (initdata or ccenabledonly):

  137.             break

  138. 

  139.     return ccenabled, creditdata, recvids, recchans, recmixes, recplayl

  140. 

  141. if __name__ == "__main__":

  142.     from sys import argv

  143.     from requests import session

  144.     from os.path import isfile

  145.     from os import environ

  146.     mysession = session()

  147.     #HSID, SSID, SID cookies required

  148.     if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  149.         cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  150.     elif isfile("config.json"):

  151.         cookies = loads(open("config.json").read())

  152.     else:

  153.         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  154.         assert False

  155.     if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  156.         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  157.         assert False

  158. 

  159.     mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  160.     vidl = argv

  161.     vidl.pop(0)

  162.     for video in vidl:

  163.         print(getmetadata(mysession, video))