tech234a
/
ytcc-archive
espelho de https://github.com/Data-Horde/ytcc-archive
1
0
Derivar 0
Código Questões 0 Lançamentos 0 Wiki Trabalho
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Não pode escolher mais do que 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
117 Cometimentos
7 Ramos
545 KiB
 
 
Árvore: d303c266be
Ramos Etiquetas
${ item.name }
Criar ramo ${ searchTerm }
de 'd303c266be'
${ noResults }
ytcc-archive/discovery.py

151 linhas
12 KiB
Em bruto Responsabilidade Histórico 

  1. from time import sleep

  2. from typing import Dict

  3. from json import loads

  4. 

  5. from switchable_request import get

  6. 

  7. backend = "http3"

  8. 

  9. langcodes = {"Afar": "aa", "Abkhazian": "ab", "Afrikaans": "af", "Akan": "ak", "all": "all", "Amharic": "am", "Aragonese": "an", "Arabic": "ar", "Aramaic": "arc", "Algerian Arabic": "arq", "Assamese": "as", "American Sign Language": "ase", "Asturian": "ast", "Avaric": "av", "Aymara": "ay", "Azerbaijani": "az", "Bashkir": "ba", "Belarusian": "be", "Bulgarian": "bg", "Bihari": "bh", "Bislama": "bi", "Bangla": "bn", "Tibetan": "bo", "Breton": "br", "Bosnian": "bs", "Catalan": "ca", "Cebuano": "ceb", "Choctaw": "cho", "Cherokee": "chr", "Corsican": "co", "Czech": "cs", "Church Slavic": "cu", "Welsh": "cy", "Danish": "da", "Danish (Denmark)": "da-DK", "German": "de", "German (Austria)": "de-AT", "German (Switzerland)": "de-CH", "German (Germany)": "de-DE", "Divehi": "dv", "Dzongkha": "dz", "Ewe": "ee", "Greek": "el", "English": "en", "English (United Arab Emirates)": "en-AE", "English (Canada)": "en-CA", "English (United Kingdom)": "en-GB", "English (Ireland)": "en-IE", "English (India)": "en-IN", "English (United States)": "en-US", "Esperanto": "eo", "Spanish": "es", "Spanish (Latin America)": "es-419", "Spanish (Argentina)": "es-AR", "Spanish (Chile)": "es-CL", "Spanish (Colombia)": "es-CO", "Spanish (Costa Rica)": "es-CR", "Spanish (Spain)": "es-ES", "Spanish (Mexico)": "es-MX", "Spanish (Nicaragua)": "es-NI", "Spanish (United States)": "es-US", "Estonian": "et", "Basque": "eu", "Persian": "fa", "Persian (Afghanistan)": "fa-AF", "Persian (Iran)": "fa-IR", "Fulah": "ff", "Finnish": "fi", "Filipino": "fil", "Fijian": "fj", "Faroese": "fo", "French": "fr", "French (Belgium)": "fr-BE", "French (Canada)": "fr-CA", "French (Switzerland)": "fr-CH", "French (France)": "fr-FR", "Western Frisian": "fy", "Irish": "ga", "Scottish Gaelic": "gd", "Galician": "gl", "Guarani": "gn", "Swiss German": "gsw", "Gujarati": "gu", "Hausa": "ha", "Hakka Chinese": "hak", "Hakka Chinese (Taiwan)": "hak-TW", "Hindi": "hi-Latn", "Hmong": "hmn", "Croatian": "hr", "Haitian Creole": "ht", "Hungarian": "hu", "Armenian": "hy", "Interlingua": "ia", "Indonesian": "id", "Interlingue": "ie", "Igbo": "ig", "Sichuan Yi": "ii", "Inupiaq": "ik", "Icelandic": "is", "Italian": "it", "Italian (Italy)": "it-IT", "Inuktitut": "iu", "Hebrew": "iw", "Japanese": "ja", "Javanese": "jv", "Georgian": "ka", "Kazakh": "kk", "Kalaallisut": "kl", "Khmer": "km", "Kannada": "kn", "Korean": "ko", "Korean (South Korea)": "ko-KR", "Kanuri": "kr", "Kashmiri": "ks", "Kurdish": "ku", "Kyrgyz": "ky", "Latin": "la", "Luxembourgish": "lb", "Lingala": "ln", "Lao": "lo", "Lithuanian": "lt", "Mizo": "lus", "Latvian": "lv", "Masai": "mas", "Malagasy": "mg", "Maori": "mi", "Miscellaneous languages": "mis", "Macedonian": "mk", "Malayalam": "ml", "Mongolian": "mn", "Manipuri": "mni", "Moldavian": "mo", "Marathi": "mr", "Malay": "ms", "Maltese": "mt", "Burmese": "my", "Nauru": "na", "Min Nan Chinese": "nan", "Min Nan Chinese (Taiwan)": "nan-TW", "Nepali": "ne", "Dutch": "nl", "Dutch (Belgium)": "nl-BE", "Dutch (Netherlands)": "nl-NL", "Norwegian Nynorsk": "nn", "Norwegian": "no", "not": "not", "Navajo": "nv", "Occitan": "oc", "Oromo": "om", "Odia": "or", "Punjabi": "pa", "Polish": "pl", "Polish (Poland)": "pl-PL", "Pashto": "ps", "Portuguese": "pt", "Portuguese (Brazil)": "pt-BR", "Portuguese (Portugal)": "pt-PT", "Quechua": "qu", "Romansh": "rm", "Rundi": "rn", "Romanian": "ro", "Romanian (Moldova)": "ro-MD", "Russian": "ru-Latn", "Russian (Russia)": "ru-RU", "Kinyarwanda": "rw", "Sanskrit": "sa", "Sardinian": "sc", "Sicilian": "scn", "Scots": "sco", "Sindhi": "sd", "Sherdukpen": "sdp", "Northern Sami": "se", "Sango": "sg", "Serbo-Croatian": "sh", "Sinhala": "si", "Slovak": "sk", "Slovenian": "sl", "Samoan": "sm", "Shona": "sn", "Somali": "so", "Albanian": "sq", "Serbian": "sr", "Serbian (Cyrillic)": "sr-Cyrl", "Serbian (Latin)": "sr-Latn", "Swati": "ss", "Southern Sotho": "st", "Sundanese": "su", "Swedish": "sv", "Swahili": "sw", "Tamil": "ta", "Telugu": "te", "Tajik": "tg", "Thai": "th", "Tigrinya": "ti", "Turkmen": "tk", "Tagalog": "tl", "Klingon": "tlh", "Tswana": "tn", "Tongan": "to", "Turkish": "tr", "Turkish (Turkey)": "tr-TR", "Tsonga": "ts", "Tatar": "tt", "Twi": "tw", "Ukrainian": "uk", "Urdu": "ur", "Uzbek": "uz", "Vietnamese": "vi", "Volap\\xFCk": "vo", "Wolof": "wo", "Xhosa": "xh", "Yiddish": "yi", "Yoruba": "yo", "Cantonese": "yue", "Cantonese (Hong Kong)": "yue-HK", "Chinese": "zh", "Chinese (China)": "zh-CN", "Chinese (Hong Kong)": "zh-HK", "Chinese (Simplified)": "zh-Hans", "Chinese (Simplified, China)": "zh-Hans-CN", "Chinese (Simplified, Singapore)": "zh-Hans-SG", "Chinese (Traditional)": "zh-Hant", "Chinese (Traditional, Hong Kong)": "zh-Hant-HK", "Chinese (Traditional, Taiwan)": "zh-Hant-TW", "Chinese (Singapore)": "zh-SG", "Chinese (Taiwan)": "zh-TW", "Zulu": "zu", "Hiri Motu": "ho", "Tok Pisin": "tpi", "Voro": "vor"}

  10. 

  11. def getmetadata(mysession, vid, allheaders):

  12.     global backend

  13.     params = (

  14.         ("v", vid),

  15.     )

  16. 

  17.     while True:

  18.         wpage = get("https://www.youtube.com/watch", params=params, mysession=mysession, backend=backend, http3headers=allheaders)

  19. 

  20.         if not """</div><div id="content" class="  content-alignment" role="main"><p class='largeText'>Sorry for the interruption. We have been receiving a large volume of requests from your network.</p>

  21. 

  22. <p>To continue with your YouTube experience, please fill out the form below.</p>""" in wpage.text and not wpage.status_code == 429 and 'window["ytInitialPlayerResponse"] = ' in wpage.text and 'window["ytInitialData"] = ' in wpage.text:

  23.             break

  24.         else:

  25.             if backend == "requests":

  26.                 backend = "http3"

  27.                 print("Captcha detected, switching discovery to HTTP3/QUIC")

  28.             else:

  29.                 print("Captcha detected, waiting 30 seconds")

  30.                 sleep(30)

  31. 

  32.     wptext = wpage.text

  33. 

  34.     initplay = None

  35.     initdata = None

  36. 

  37.     recvids  = set()

  38.     recchans = set()

  39.     recmixes = set()

  40.     recplayl = set()

  41. 

  42.     ccenabled = False #default values

  43.     creditdata = {}

  44. 

  45.     for line in wptext.splitlines():

  46.         if line.strip().startswith('window["ytInitialPlayerResponse"] = '):

  47.             initplay = loads(line.split('window["ytInitialPlayerResponse"] = ', 1)[1].strip()[:-1])

  48. 

  49.             if initplay["playabilityStatus"]["status"] == "ERROR":

  50.                 print(vid, "unavailable")

  51.                 return False, {}, recvids, recchans, recmixes, recplayl

  52.             

  53.             if "endscreen" in initplay.keys():

  54.                 if "endscreenRenderer" in initplay["endscreen"].keys():

  55.                     for el in initplay["endscreen"]["endscreenRenderer"]:

  56. 

  57.                         if type(el) == Dict:

  58.                             elint = el["endscreenElementRenderer"]

  59. 

  60.                             if "endscreenElementRenderer" in el.keys():

  61.                                 if elint["style"] == "VIDEO":

  62.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  63. 

  64.                                 elif elint["style"] == "CHANNEL":

  65.                                     try:

  66.                                         recchans.add(elint["endpoint"]["browseEndpoint"]["browseId"])

  67.                                     except:

  68.                                         print("Channel endscreen error")

  69.                                         raise

  70. 

  71.                                 elif elint["style"] == "PLAYLIST":

  72.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  73.                                     recplayl.add(elint["endpoint"]["watchEndpint"]["playlistId"])

  74. 

  75.             if "captions" in initplay.keys():

  76.                 ccenabled = "contribute" in initplay["captions"]["playerCaptionsRenderer"]

  77.             else:

  78.                 ccenabled = False # if captions information is not present, community contributions are not enabled

  79. 

  80.             if "videoDetails" in initplay.keys():

  81.                 if "channelId" in initplay["videoDetails"].keys():

  82.                     recchans.add(initplay["videoDetails"]["channelId"])

  83.         elif line.strip().startswith('window["ytInitialData"] = '):

  84.             initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  85.             if "contents" in initdata.keys(): #prevent exception

  86.                 try:

  87.                     if "results" in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"].keys():

  88.                         for recmd in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"]["results"]:

  89.                             #auto is like the others

  90.                             if "compactAutoplayRenderer" in recmd.keys():

  91.                                 recmd = recmd["compactAutoplayRenderer"]["contents"][0]

  92. 

  93.                             if "compactVideoRenderer" in recmd.keys():

  94.                                 recvids.add(recmd["compactVideoRenderer"]["videoId"])

  95.                                 try:

  96.                                     recchans.add(recmd["compactVideoRenderer"]["channelId"])

  97.                                 except KeyError as e:

  98.                                     try:

  99.                                         recchans.add(recmd["compactVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  100.                                     except KeyError as e:

  101.                                         print("Channel extract error")

  102.                                     #raise

  103.                                     #print("Unable to extract channel:")

  104.                                     #print(recmd["compactVideoRenderer"])

  105. 

  106.                             elif "compactPlaylistRenderer" in recmd.keys():

  107.                                 recplayl.add(recmd["compactPlaylistRenderer"]["playlistId"])

  108.                                 if "navigationEndpoint" in recmd["compactPlaylistRenderer"].keys():

  109.                                     recvids.add(recmd["compactPlaylistRenderer"]["navigationEndpoint"]["watchEndpoint"]["videoId"])

  110.                                 if "navigationEndpoint" in recmd["compactPlaylistRenderer"]["shortBylineText"].keys():

  111.                                     recchans.add(recmd["compactPlaylistRenderer"]["shortBylineText"]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  112. 

  113.                             elif "compactRadioRenderer" in recmd.keys(): #mix playlist

  114.                                 recmixes.add(recmd["compactRadioRenderer"]["playlistId"])

  115.                             # todo: find out if channels can be suggested

  116.                 except BaseException as e:

  117.                     print(e)

  118.                     print("Exception in discovery, continuing anyway")

  119. 

  120.             creditdata = {}

  121.             try:

  122.                 mdinfo = initdata["contents"]["twoColumnWatchNextResults"]["results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"]["metadataRowContainer"]["metadataRowContainerRenderer"]["rows"]

  123.                 for item in mdinfo:

  124.                     if item["metadataRowRenderer"]["title"]["simpleText"].startswith("Caption author"): #the request to /watch needs to be in English for this to work

  125.                         try:

  126.                             desl = langcodes[item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]]

  127.                         except KeyError as e:

  128.                             #print(e)

  129.                             print("Language code conversion error, using language name")

  130.                             desl = item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]

  131.                         creditdata[desl] = []

  132.                         for itemint in item["metadataRowRenderer"]["contents"]:

  133.                             creditdata[desl].append({"name": itemint["runs"][0]["text"], "channel": itemint["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]})

  134. 

  135.             except KeyError as e:

  136.                 #print("Video does not have credits")

  137.                 pass

  138.                 #raise

  139.                 #print(e)

  140.         

  141.         if initplay and initdata:

  142.             break

  143. 

  144.     return ccenabled, creditdata, recvids, recchans, recmixes, recplayl

  145. 

  146. if __name__ == "__main__":

  147.     from sys import argv

  148.     vidl = argv

  149.     vidl.pop(0)

  150.     for video in vidl:

  151.         print(getmetadata(video))