tech234a
/
ytcc-archive
espelhamento de https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Código Issues 0 Versões 0 Wiki Atividade
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Você não pode selecionar mais de 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
129 Commits
7 Branches
162 KiB
 
 
Branch: http3
Branches Tags
${ item.name }
Criar branch ${ searchTerm }
de http3
${ noResults }
ytcc-archive/discovery.py

157 linhas
13 KiB
Original Link permanente Anotar Histórico 

  1. from time import sleep

  2. from typing import Dict

  3. from json import loads

  4. 

  5. from switchable_request import get

  6. 

  7. backend = "requests"

  8. failcnt = 0

  9. 

  10. langcodes = {"Afar": "aa", "Abkhazian": "ab", "Afrikaans": "af", "Akan": "ak", "all": "all", "Amharic": "am", "Aragonese": "an", "Arabic": "ar", "Aramaic": "arc", "Algerian Arabic": "arq", "Assamese": "as", "American Sign Language": "ase", "Asturian": "ast", "Avaric": "av", "Aymara": "ay", "Azerbaijani": "az", "Bashkir": "ba", "Belarusian": "be", "Bulgarian": "bg", "Bihari": "bh", "Bislama": "bi", "Bangla": "bn", "Tibetan": "bo", "Breton": "br", "Bosnian": "bs", "Catalan": "ca", "Cebuano": "ceb", "Choctaw": "cho", "Cherokee": "chr", "Corsican": "co", "Czech": "cs", "Church Slavic": "cu", "Welsh": "cy", "Danish": "da", "Danish (Denmark)": "da-DK", "German": "de", "German (Austria)": "de-AT", "German (Switzerland)": "de-CH", "German (Germany)": "de-DE", "Divehi": "dv", "Dzongkha": "dz", "Ewe": "ee", "Greek": "el", "English": "en", "English (United Arab Emirates)": "en-AE", "English (Canada)": "en-CA", "English (United Kingdom)": "en-GB", "English (Ireland)": "en-IE", "English (India)": "en-IN", "English (United States)": "en-US", "Esperanto": "eo", "Spanish": "es", "Spanish (Latin America)": "es-419", "Spanish (Argentina)": "es-AR", "Spanish (Chile)": "es-CL", "Spanish (Colombia)": "es-CO", "Spanish (Costa Rica)": "es-CR", "Spanish (Spain)": "es-ES", "Spanish (Mexico)": "es-MX", "Spanish (Nicaragua)": "es-NI", "Spanish (United States)": "es-US", "Estonian": "et", "Basque": "eu", "Persian": "fa", "Persian (Afghanistan)": "fa-AF", "Persian (Iran)": "fa-IR", "Fulah": "ff", "Finnish": "fi", "Filipino": "fil", "Fijian": "fj", "Faroese": "fo", "French": "fr", "French (Belgium)": "fr-BE", "French (Canada)": "fr-CA", "French (Switzerland)": "fr-CH", "French (France)": "fr-FR", "Western Frisian": "fy", "Irish": "ga", "Scottish Gaelic": "gd", "Galician": "gl", "Guarani": "gn", "Swiss German": "gsw", "Gujarati": "gu", "Hausa": "ha", "Hakka Chinese": "hak", "Hakka Chinese (Taiwan)": "hak-TW", "Hindi": "hi-Latn", "Hmong": "hmn", "Croatian": "hr", "Haitian Creole": "ht", "Hungarian": "hu", "Armenian": "hy", "Interlingua": "ia", "Indonesian": "id", "Interlingue": "ie", "Igbo": "ig", "Sichuan Yi": "ii", "Inupiaq": "ik", "Icelandic": "is", "Italian": "it", "Italian (Italy)": "it-IT", "Inuktitut": "iu", "Hebrew": "iw", "Japanese": "ja", "Javanese": "jv", "Georgian": "ka", "Kazakh": "kk", "Kalaallisut": "kl", "Khmer": "km", "Kannada": "kn", "Korean": "ko", "Korean (South Korea)": "ko-KR", "Kanuri": "kr", "Kashmiri": "ks", "Kurdish": "ku", "Kyrgyz": "ky", "Latin": "la", "Luxembourgish": "lb", "Lingala": "ln", "Lao": "lo", "Lithuanian": "lt", "Mizo": "lus", "Latvian": "lv", "Masai": "mas", "Malagasy": "mg", "Maori": "mi", "Miscellaneous languages": "mis", "Macedonian": "mk", "Malayalam": "ml", "Mongolian": "mn", "Manipuri": "mni", "Moldavian": "mo", "Marathi": "mr", "Malay": "ms", "Maltese": "mt", "Burmese": "my", "Nauru": "na", "Min Nan Chinese": "nan", "Min Nan Chinese (Taiwan)": "nan-TW", "Nepali": "ne", "Dutch": "nl", "Dutch (Belgium)": "nl-BE", "Dutch (Netherlands)": "nl-NL", "Norwegian Nynorsk": "nn", "Norwegian": "no", "not": "not", "Navajo": "nv", "Occitan": "oc", "Oromo": "om", "Odia": "or", "Punjabi": "pa", "Polish": "pl", "Polish (Poland)": "pl-PL", "Pashto": "ps", "Portuguese": "pt", "Portuguese (Brazil)": "pt-BR", "Portuguese (Portugal)": "pt-PT", "Quechua": "qu", "Romansh": "rm", "Rundi": "rn", "Romanian": "ro", "Romanian (Moldova)": "ro-MD", "Russian": "ru-Latn", "Russian (Russia)": "ru-RU", "Kinyarwanda": "rw", "Sanskrit": "sa", "Sardinian": "sc", "Sicilian": "scn", "Scots": "sco", "Sindhi": "sd", "Sherdukpen": "sdp", "Northern Sami": "se", "Sango": "sg", "Serbo-Croatian": "sh", "Sinhala": "si", "Slovak": "sk", "Slovenian": "sl", "Samoan": "sm", "Shona": "sn", "Somali": "so", "Albanian": "sq", "Serbian": "sr", "Serbian (Cyrillic)": "sr-Cyrl", "Serbian (Latin)": "sr-Latn", "Swati": "ss", "Southern Sotho": "st", "Sundanese": "su", "Swedish": "sv", "Swahili": "sw", "Tamil": "ta", "Telugu": "te", "Tajik": "tg", "Thai": "th", "Tigrinya": "ti", "Turkmen": "tk", "Tagalog": "tl", "Klingon": "tlh", "Tswana": "tn", "Tongan": "to", "Turkish": "tr", "Turkish (Turkey)": "tr-TR", "Tsonga": "ts", "Tatar": "tt", "Twi": "tw", "Ukrainian": "uk", "Urdu": "ur", "Uzbek": "uz", "Vietnamese": "vi", "Volap\\xFCk": "vo", "Wolof": "wo", "Xhosa": "xh", "Yiddish": "yi", "Yoruba": "yo", "Cantonese": "yue", "Cantonese (Hong Kong)": "yue-HK", "Chinese": "zh", "Chinese (China)": "zh-CN", "Chinese (Hong Kong)": "zh-HK", "Chinese (Simplified)": "zh-Hans", "Chinese (Simplified, China)": "zh-Hans-CN", "Chinese (Simplified, Singapore)": "zh-Hans-SG", "Chinese (Traditional)": "zh-Hant", "Chinese (Traditional, Hong Kong)": "zh-Hant-HK", "Chinese (Traditional, Taiwan)": "zh-Hant-TW", "Chinese (Singapore)": "zh-SG", "Chinese (Taiwan)": "zh-TW", "Zulu": "zu", "Hiri Motu": "ho", "Tok Pisin": "tpi", "Voro": "vor"}

  11. 

  12. def getmetadata(mysession, vid, allheaders):

  13.     global backend

  14.     global failcnt

  15.     params = (

  16.         ("v", vid),

  17.     )

  18. 

  19.     while True:

  20.         wpage = get("https://www.youtube.com/watch", params=params, mysession=mysession, backend=backend, http3headers=allheaders)

  21. 

  22.         if not """</div><div id="content" class="  content-alignment" role="main"><p class='largeText'>Sorry for the interruption. We have been receiving a large volume of requests from your network.</p>

  23. 

  24. <p>To continue with your YouTube experience, please fill out the form below.</p>""" in wpage.text and not wpage.status_code == 429 and 'window["ytInitialPlayerResponse"] = ' in wpage.text and 'window["ytInitialData"] = ' in wpage.text:

  25.             break

  26.         else:

  27.             if backend == "requests" and failcnt > 30:

  28.                 backend = "http3"

  29.                 print("Captcha detected, switching discovery to HTTP3/QUIC")

  30.             elif backend == "http3" and failcnt < 30:

  31.                 failcnt += 1

  32.                 print("Captcha detected, waiting 30 seconds... ", 30-failcnt, "attempts left until switching discovery to HTTP3/QUIC.")

  33.                 sleep(30)

  34.             else:

  35.                 print("Captcha detected, waiting 30 seconds")

  36.                 sleep(30)

  37. 

  38.     wptext = wpage.text

  39. 

  40.     initplay = None

  41.     initdata = None

  42. 

  43.     recvids  = set()

  44.     recchans = set()

  45.     recmixes = set()

  46.     recplayl = set()

  47. 

  48.     ccenabled = False #default values

  49.     creditdata = {}

  50. 

  51.     for line in wptext.splitlines():

  52.         if line.strip().startswith('window["ytInitialPlayerResponse"] = '):

  53.             initplay = loads(line.split('window["ytInitialPlayerResponse"] = ', 1)[1].strip()[:-1])

  54. 

  55.             if initplay["playabilityStatus"]["status"] == "ERROR":

  56.                 print(vid, "unavailable")

  57.                 return False, {}, recvids, recchans, recmixes, recplayl

  58.             

  59.             if "endscreen" in initplay.keys():

  60.                 if "endscreenRenderer" in initplay["endscreen"].keys():

  61.                     for el in initplay["endscreen"]["endscreenRenderer"]:

  62. 

  63.                         if type(el) == Dict:

  64.                             elint = el["endscreenElementRenderer"]

  65. 

  66.                             if "endscreenElementRenderer" in el.keys():

  67.                                 if elint["style"] == "VIDEO":

  68.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  69. 

  70.                                 elif elint["style"] == "CHANNEL":

  71.                                     try:

  72.                                         recchans.add(elint["endpoint"]["browseEndpoint"]["browseId"])

  73.                                     except:

  74.                                         print("Channel endscreen error")

  75.                                         raise

  76. 

  77.                                 elif elint["style"] == "PLAYLIST":

  78.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  79.                                     recplayl.add(elint["endpoint"]["watchEndpint"]["playlistId"])

  80. 

  81.             if "captions" in initplay.keys():

  82.                 ccenabled = "contribute" in initplay["captions"]["playerCaptionsRenderer"]

  83.             else:

  84.                 ccenabled = False # if captions information is not present, community contributions are not enabled

  85. 

  86.             if "videoDetails" in initplay.keys():

  87.                 if "channelId" in initplay["videoDetails"].keys():

  88.                     recchans.add(initplay["videoDetails"]["channelId"])

  89.         elif line.strip().startswith('window["ytInitialData"] = '):

  90.             initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  91.             if "contents" in initdata.keys(): #prevent exception

  92.                 try:

  93.                     if "results" in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"].keys():

  94.                         for recmd in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"]["results"]:

  95.                             #auto is like the others

  96.                             if "compactAutoplayRenderer" in recmd.keys():

  97.                                 recmd = recmd["compactAutoplayRenderer"]["contents"][0]

  98. 

  99.                             if "compactVideoRenderer" in recmd.keys():

  100.                                 recvids.add(recmd["compactVideoRenderer"]["videoId"])

  101.                                 try:

  102.                                     recchans.add(recmd["compactVideoRenderer"]["channelId"])

  103.                                 except KeyError as e:

  104.                                     try:

  105.                                         recchans.add(recmd["compactVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  106.                                     except KeyError as e:

  107.                                         print("Channel extract error")

  108.                                     #raise

  109.                                     #print("Unable to extract channel:")

  110.                                     #print(recmd["compactVideoRenderer"])

  111. 

  112.                             elif "compactPlaylistRenderer" in recmd.keys():

  113.                                 recplayl.add(recmd["compactPlaylistRenderer"]["playlistId"])

  114.                                 if "navigationEndpoint" in recmd["compactPlaylistRenderer"].keys():

  115.                                     recvids.add(recmd["compactPlaylistRenderer"]["navigationEndpoint"]["watchEndpoint"]["videoId"])

  116.                                 if "navigationEndpoint" in recmd["compactPlaylistRenderer"]["shortBylineText"].keys():

  117.                                     recchans.add(recmd["compactPlaylistRenderer"]["shortBylineText"]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  118. 

  119.                             elif "compactRadioRenderer" in recmd.keys(): #mix playlist

  120.                                 recmixes.add(recmd["compactRadioRenderer"]["playlistId"])

  121.                             # todo: find out if channels can be suggested

  122.                 except BaseException as e:

  123.                     print(e)

  124.                     print("Exception in discovery, continuing anyway")

  125. 

  126.             creditdata = {}

  127.             try:

  128.                 mdinfo = initdata["contents"]["twoColumnWatchNextResults"]["results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"]["metadataRowContainer"]["metadataRowContainerRenderer"]["rows"]

  129.                 for item in mdinfo:

  130.                     if item["metadataRowRenderer"]["title"]["simpleText"].startswith("Caption author"): #the request to /watch needs to be in English for this to work

  131.                         try:

  132.                             desl = langcodes[item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]]

  133.                         except KeyError as e:

  134.                             #print(e)

  135.                             print("Language code conversion error, using language name")

  136.                             desl = item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]

  137.                         creditdata[desl] = []

  138.                         for itemint in item["metadataRowRenderer"]["contents"]:

  139.                             creditdata[desl].append({"name": itemint["runs"][0]["text"], "channel": itemint["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]})

  140. 

  141.             except KeyError as e:

  142.                 #print("Video does not have credits")

  143.                 pass

  144.                 #raise

  145.                 #print(e)

  146.         

  147.         if initplay and initdata:

  148.             break

  149. 

  150.     return ccenabled, creditdata, recvids, recchans, recmixes, recplayl

  151. 

  152. if __name__ == "__main__":

  153.     from sys import argv

  154.     vidl = argv

  155.     vidl.pop(0)

  156.     for video in vidl:

  157.         print(getmetadata(video))