tech234a
/
ytcc-archive
espelhamento de https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Código Issues 0 Versões 0 Wiki Atividade
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Você não pode selecionar mais de 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
154 Commits
7 Branches
179 KiB
 
 
Branch: master
Branches Tags
${ item.name }
Criar branch ${ searchTerm }
de master
${ noResults }
ytcc-archive/discovery.py

154 linhas
13 KiB
Original Link permanente Anotar Histórico 

  1. from time import sleep

  2. from typing import Dict

  3. from json import loads

  4. 

  5. langcodes = {"Afar": "aa", "Abkhazian": "ab", "Afrikaans": "af", "Akan": "ak", "all": "all", "Amharic": "am", "Aragonese": "an", "Arabic": "ar", "Aramaic": "arc", "Algerian Arabic": "arq", "Assamese": "as", "American Sign Language": "ase", "Asturian": "ast", "Avaric": "av", "Aymara": "ay", "Azerbaijani": "az", "Bashkir": "ba", "Belarusian": "be", "Bulgarian": "bg", "Bihari": "bh", "Bislama": "bi", "Bangla": "bn", "Tibetan": "bo", "Breton": "br", "Bosnian": "bs", "Catalan": "ca", "Cebuano": "ceb", "Choctaw": "cho", "Cherokee": "chr", "Corsican": "co", "Czech": "cs", "Church Slavic": "cu", "Welsh": "cy", "Danish": "da", "Danish (Denmark)": "da-DK", "German": "de", "German (Austria)": "de-AT", "German (Switzerland)": "de-CH", "German (Germany)": "de-DE", "Divehi": "dv", "Dzongkha": "dz", "Ewe": "ee", "Greek": "el", "English": "en", "English (United Arab Emirates)": "en-AE", "English (Canada)": "en-CA", "English (United Kingdom)": "en-GB", "English (Ireland)": "en-IE", "English (India)": "en-IN", "English (United States)": "en-US", "Esperanto": "eo", "Spanish": "es", "Spanish (Latin America)": "es-419", "Spanish (Argentina)": "es-AR", "Spanish (Chile)": "es-CL", "Spanish (Colombia)": "es-CO", "Spanish (Costa Rica)": "es-CR", "Spanish (Spain)": "es-ES", "Spanish (Mexico)": "es-MX", "Spanish (Nicaragua)": "es-NI", "Spanish (United States)": "es-US", "Estonian": "et", "Basque": "eu", "Persian": "fa", "Persian (Afghanistan)": "fa-AF", "Persian (Iran)": "fa-IR", "Fulah": "ff", "Finnish": "fi", "Filipino": "fil", "Fijian": "fj", "Faroese": "fo", "French": "fr", "French (Belgium)": "fr-BE", "French (Canada)": "fr-CA", "French (Switzerland)": "fr-CH", "French (France)": "fr-FR", "Western Frisian": "fy", "Irish": "ga", "Scottish Gaelic": "gd", "Galician": "gl", "Guarani": "gn", "Swiss German": "gsw", "Gujarati": "gu", "Hausa": "ha", "Hakka Chinese": "hak", "Hakka Chinese (Taiwan)": "hak-TW", "Hindi": "hi-Latn", "Hmong": "hmn", "Croatian": "hr", "Haitian Creole": "ht", "Hungarian": "hu", "Armenian": "hy", "Interlingua": "ia", "Indonesian": "id", "Interlingue": "ie", "Igbo": "ig", "Sichuan Yi": "ii", "Inupiaq": "ik", "Icelandic": "is", "Italian": "it", "Italian (Italy)": "it-IT", "Inuktitut": "iu", "Hebrew": "iw", "Japanese": "ja", "Javanese": "jv", "Georgian": "ka", "Kazakh": "kk", "Kalaallisut": "kl", "Khmer": "km", "Kannada": "kn", "Korean": "ko", "Korean (South Korea)": "ko-KR", "Kanuri": "kr", "Kashmiri": "ks", "Kurdish": "ku", "Kyrgyz": "ky", "Latin": "la", "Luxembourgish": "lb", "Lingala": "ln", "Lao": "lo", "Lithuanian": "lt", "Mizo": "lus", "Latvian": "lv", "Masai": "mas", "Malagasy": "mg", "Maori": "mi", "Miscellaneous languages": "mis", "Macedonian": "mk", "Malayalam": "ml", "Mongolian": "mn", "Manipuri": "mni", "Moldavian": "mo", "Marathi": "mr", "Malay": "ms", "Maltese": "mt", "Burmese": "my", "Nauru": "na", "Min Nan Chinese": "nan", "Min Nan Chinese (Taiwan)": "nan-TW", "Nepali": "ne", "Dutch": "nl", "Dutch (Belgium)": "nl-BE", "Dutch (Netherlands)": "nl-NL", "Norwegian Nynorsk": "nn", "Norwegian": "no", "not": "not", "Navajo": "nv", "Occitan": "oc", "Oromo": "om", "Odia": "or", "Punjabi": "pa", "Polish": "pl", "Polish (Poland)": "pl-PL", "Pashto": "ps", "Portuguese": "pt", "Portuguese (Brazil)": "pt-BR", "Portuguese (Portugal)": "pt-PT", "Quechua": "qu", "Romansh": "rm", "Rundi": "rn", "Romanian": "ro", "Romanian (Moldova)": "ro-MD", "Russian": "ru-Latn", "Russian (Russia)": "ru-RU", "Kinyarwanda": "rw", "Sanskrit": "sa", "Sardinian": "sc", "Sicilian": "scn", "Scots": "sco", "Sindhi": "sd", "Sherdukpen": "sdp", "Northern Sami": "se", "Sango": "sg", "Serbo-Croatian": "sh", "Sinhala": "si", "Slovak": "sk", "Slovenian": "sl", "Samoan": "sm", "Shona": "sn", "Somali": "so", "Albanian": "sq", "Serbian": "sr", "Serbian (Cyrillic)": "sr-Cyrl", "Serbian (Latin)": "sr-Latn", "Swati": "ss", "Southern Sotho": "st", "Sundanese": "su", "Swedish": "sv", "Swahili": "sw", "Tamil": "ta", "Telugu": "te", "Tajik": "tg", "Thai": "th", "Tigrinya": "ti", "Turkmen": "tk", "Tagalog": "tl", "Klingon": "tlh", "Tswana": "tn", "Tongan": "to", "Turkish": "tr", "Turkish (Turkey)": "tr-TR", "Tsonga": "ts", "Tatar": "tt", "Twi": "tw", "Ukrainian": "uk", "Urdu": "ur", "Uzbek": "uz", "Vietnamese": "vi", "Volap\\xFCk": "vo", "Wolof": "wo", "Xhosa": "xh", "Yiddish": "yi", "Yoruba": "yo", "Cantonese": "yue", "Cantonese (Hong Kong)": "yue-HK", "Chinese": "zh", "Chinese (China)": "zh-CN", "Chinese (Hong Kong)": "zh-HK", "Chinese (Simplified)": "zh-Hans", "Chinese (Simplified, China)": "zh-Hans-CN", "Chinese (Simplified, Singapore)": "zh-Hans-SG", "Chinese (Traditional)": "zh-Hant", "Chinese (Traditional, Hong Kong)": "zh-Hant-HK", "Chinese (Traditional, Taiwan)": "zh-Hant-TW", "Chinese (Singapore)": "zh-SG", "Chinese (Taiwan)": "zh-TW", "Zulu": "zu", "Hiri Motu": "ho", "Tok Pisin": "tpi", "Voro": "vor"}

  6. 

  7. def getmetadata(mysession, vid, ccenabledonly=False):

  8.     params = (

  9.         ("v", vid),

  10.     )

  11. 

  12.     while True:

  13.         wpage = mysession.get("https://www.youtube.com/watch", params=params)

  14.         if not """</div><div id="content" class="  content-alignment" role="main"><p class='largeText'>Sorry for the interruption. We have been receiving a large volume of requests from your network.</p>

  15. 

  16. <p>To continue with your YouTube experience, please fill out the form below.</p>""" in wpage.text and not wpage.status_code == 429 and 'window["ytInitialPlayerResponse"] = ' in wpage.text and 'window["ytInitialData"] = ' in wpage.text:

  17.             break

  18.         else:

  19.             print("Captcha detected, waiting 30 seconds")

  20.             sleep(30)

  21. 

  22.     wptext = wpage.text

  23. 

  24.     initplay = None

  25.     initdata = None

  26. 

  27.     recvids  = set()

  28.     recchans = set()

  29.     recmixes = set()

  30.     recplayl = set()

  31. 

  32.     pparams = (

  33.                 ("v", vid),

  34.                 #("lang", langcode),

  35.                 ("action_mde_edit_form", 1),

  36.                 ("bl", "vmp"),

  37.                 ("ui", "hd"),

  38.                 ("tab", "captions"),

  39.                 #('forceedit', 'captions'),

  40.                 ("lang","aa")

  41.             )

  42.     #mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  43.     ccenabled = not "yet enabled community" in mysession.get("https://www.youtube.com/timedtext_editor", params=pparams).text

  44.     #print(mysession.get("https://www.youtube.com/timedtext_editor", params=pparams).text)

  45.     creditdata = {}

  46. 

  47.     for line in wptext.splitlines():

  48.         if line.strip().startswith('window["ytInitialPlayerResponse"] = '):

  49.             initplay = loads(line.split('window["ytInitialPlayerResponse"] = ', 1)[1].strip()[:-1])

  50. 

  51.             if not ccenabledonly:

  52.                 if initplay["playabilityStatus"]["status"] == "ERROR":

  53.                     print(vid, "unavailable")

  54.                     return False, {}, recvids, recchans, recmixes, recplayl

  55.                 

  56.                 if "endscreen" in initplay.keys():

  57.                     if "endscreenRenderer" in initplay["endscreen"].keys():

  58.                         for el in initplay["endscreen"]["endscreenRenderer"]:

  59. 

  60.                             if type(el) == Dict:

  61.                                 elint = el["endscreenElementRenderer"]

  62. 

  63.                                 if "endscreenElementRenderer" in el.keys():

  64.                                     if elint["style"] == "VIDEO":

  65.                                         recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  66. 

  67.                                     elif elint["style"] == "CHANNEL":

  68.                                         try:

  69.                                             recchans.add(elint["endpoint"]["browseEndpoint"]["browseId"])

  70.                                         except:

  71.                                             print("Channel endscreen error")

  72.                                             raise

  73. 

  74.                                     elif elint["style"] == "PLAYLIST":

  75.                                         recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  76.                                         recplayl.add(elint["endpoint"]["watchEndpint"]["playlistId"])

  77. 

  78.             """

  79.             if "captions" in initplay.keys():

  80.                 ccenabled = "contribute" in initplay["captions"]["playerCaptionsRenderer"]

  81.             else:

  82.                 ccenabled = False # if captions information is not present, community contributions are not enabled

  83.             """

  84. 

  85.             if not ccenabledonly:

  86.                 if "videoDetails" in initplay.keys():

  87.                     if "channelId" in initplay["videoDetails"].keys():

  88.                         recchans.add(initplay["videoDetails"]["channelId"])

  89.         elif line.strip().startswith('window["ytInitialData"] = '):

  90.             if not ccenabledonly:

  91.                 initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  92.                 if "contents" in initdata.keys(): #prevent exception

  93.                     try:

  94.                         if "results" in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"].keys():

  95.                             for recmd in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"]["results"]:

  96.                                 #auto is like the others

  97.                                 if "compactAutoplayRenderer" in recmd.keys():

  98.                                     recmd = recmd["compactAutoplayRenderer"]["contents"][0]

  99. 

  100.                                 if "compactVideoRenderer" in recmd.keys():

  101.                                     recvids.add(recmd["compactVideoRenderer"]["videoId"])

  102.                                     try:

  103.                                         recchans.add(recmd["compactVideoRenderer"]["channelId"])

  104.                                     except KeyError as e:

  105.                                         try:

  106.                                             recchans.add(recmd["compactVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  107.                                         except KeyError as e:

  108.                                             print("Channel extract error")

  109.                                         #raise

  110.                                         #print("Unable to extract channel:")

  111.                                         #print(recmd["compactVideoRenderer"])

  112. 

  113.                                 elif "compactPlaylistRenderer" in recmd.keys():

  114.                                     recplayl.add(recmd["compactPlaylistRenderer"]["playlistId"])

  115.                                     if "navigationEndpoint" in recmd["compactPlaylistRenderer"].keys():

  116.                                         recvids.add(recmd["compactPlaylistRenderer"]["navigationEndpoint"]["watchEndpoint"]["videoId"])

  117.                                     if "navigationEndpoint" in recmd["compactPlaylistRenderer"]["shortBylineText"].keys():

  118.                                         recchans.add(recmd["compactPlaylistRenderer"]["shortBylineText"]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  119. 

  120.                                 elif "compactRadioRenderer" in recmd.keys(): #mix playlist

  121.                                     recmixes.add(recmd["compactRadioRenderer"]["playlistId"])

  122.                                 # todo: find out if channels can be suggested

  123.                     except BaseException as e:

  124.                         print(e)

  125.                         print("Exception in discovery, continuing anyway")

  126.         

  127.         if initplay and (initdata or ccenabledonly):

  128.             break

  129. 

  130.     return ccenabled, recvids, recchans, recmixes, recplayl

  131. 

  132. if __name__ == "__main__":

  133.     from sys import argv

  134.     from requests import session

  135.     from os.path import isfile

  136.     from os import environ

  137.     mysession = session()

  138.     #HSID, SSID, SID cookies required

  139.     if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  140.         cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  141.     elif isfile("config.json"):

  142.         cookies = loads(open("config.json").read())

  143.     else:

  144.         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  145.         assert False

  146.     if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  147.         print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  148.         assert False

  149. 

  150.     mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  151.     vidl = argv

  152.     vidl.pop(0)

  153.     for video in vidl:

  154.         print(getmetadata(mysession, video))