tech234a
3 years ago
1 changed files with 39 additions and 79 deletions
Unified View
Diff Options
-
+39 -79worker.py
+ 39
- 79
worker.py
View File
| @@ -3,7 +3,7 @@ import requests | |||||
| from time import sleep | from time import sleep | ||||
| from os import mkdir, rmdir, listdir, system, environ | from os import mkdir, rmdir, listdir, system, environ | ||||
| from os.path import isdir, isfile, getsize | from os.path import isdir, isfile, getsize | ||||
| from json import dumps, loads | |||||
| from json import loads | |||||
| from youtube_channel import main | from youtube_channel import main | ||||
| @@ -87,7 +87,6 @@ open("cookies.txt", "w").write("""# HTTP Cookie File | |||||
| del cookies | del cookies | ||||
| validationtimes = 0 | validationtimes = 0 | ||||
| shouldgetjob = True | |||||
| #Graceful Shutdown | #Graceful Shutdown | ||||
| class GracefulKiller: | class GracefulKiller: | ||||
| @@ -102,20 +101,8 @@ class GracefulKiller: | |||||
| gkiller = GracefulKiller() | gkiller = GracefulKiller() | ||||
| #REMOVED PANIC MECHANISM! | |||||
| """ | |||||
| enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0] | |||||
| if not enres: | |||||
| print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.") | |||||
| shouldgetjob = False | |||||
| gkiller.kill_now = True #exit the script | |||||
| del enres | |||||
| """ | |||||
| #microtasks | #microtasks | ||||
| def threadrunner(): | def threadrunner(): | ||||
| global shouldgetjob | |||||
| global validationtimes | global validationtimes | ||||
| jobs = Queue() | jobs = Queue() | ||||
| ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False}) | ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False}) | ||||
| @@ -124,7 +111,6 @@ def threadrunner(): | |||||
| task, vid, args = jobs.get() | task, vid, args = jobs.get() | ||||
| if task == "submitdiscovery": | if task == "submitdiscovery": | ||||
| tracker.add_item_to_tracker(args, vid) | tracker.add_item_to_tracker(args, vid) | ||||
| #pass | |||||
| elif task == "discovery": | elif task == "discovery": | ||||
| while True: | while True: | ||||
| @@ -159,9 +145,6 @@ def threadrunner(): | |||||
| jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist)) | jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist)) | ||||
| for playldisc in info[4]: | for playldisc in info[4]: | ||||
| jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist)) | jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist)) | ||||
| #jobs.put(("complete", None, "video:"+vid)) | |||||
| #pass | |||||
| elif task == "subtitles": | elif task == "subtitles": | ||||
| subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions) | subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions) | ||||
| @@ -198,17 +181,14 @@ def threadrunner(): | |||||
| elif task == "mixplaylist": | elif task == "mixplaylist": | ||||
| try: | try: | ||||
| wptext = mysession.get("https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]).text | wptext = mysession.get("https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]).text | ||||
| #chanl = set() | |||||
| #channel handling not needed here because we will get it from the video | #channel handling not needed here because we will get it from the video | ||||
| for line in wptext.splitlines(): | for line in wptext.splitlines(): | ||||
| if line.strip().startswith('window["ytInitialData"] = '): | if line.strip().startswith('window["ytInitialData"] = '): | ||||
| initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1]) | initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1]) | ||||
| for itemyvp in initdata["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"]["contents"]: | for itemyvp in initdata["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"]["contents"]: | ||||
| jobs.put(("submitdiscovery", itemyvp["playlistPanelVideoRenderer"]["videoId"], tracker.ItemType.Video)) | jobs.put(("submitdiscovery", itemyvp["playlistPanelVideoRenderer"]["videoId"], tracker.ItemType.Video)) | ||||
| #chanl.add(itemyvp["playlistPanelVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]) | |||||
| #for itemn in chanl: | |||||
| # jobs.put(("submitdiscovery", itemn, tracker.ItemType.Channel)) | |||||
| jobs.put(("complete", None, "mixplaylist:"+args)) | jobs.put(("complete", None, "mixplaylist:"+args)) | ||||
| except: | except: | ||||
| print("Mix Playlist error, ignoring but not marking as complete...", "https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]) | print("Mix Playlist error, ignoring but not marking as complete...", "https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]) | ||||
| @@ -262,60 +242,44 @@ def threadrunner(): | |||||
| # get a new task from tracker | # get a new task from tracker | ||||
| collect() #cleanup | collect() #cleanup | ||||
| #Protection Mechanism Disarmed | |||||
| """ | |||||
| #check that the account has community contributions enabled every 50th item | |||||
| validationtimes += 1 | |||||
| if not validationtimes % 50: | |||||
| enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0] | |||||
| if not enres: | |||||
| print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.") | |||||
| shouldgetjob = False | |||||
| gkiller.kill_now = True #exit the script | |||||
| del enres | |||||
| """ | |||||
| if shouldgetjob: | |||||
| desit = tracker.request_item_from_tracker() | |||||
| print("New task:", desit) | |||||
| if desit: | |||||
| if desit.split(":", 1)[0] == "video": | |||||
| needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, | |||||
| 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, | |||||
| 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, | |||||
| 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, | |||||
| 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, | |||||
| 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, | |||||
| 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, | |||||
| 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, | |||||
| 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, | |||||
| 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, | |||||
| 'xh': None, 'yi': None, 'yo': None, 'zu': None} | |||||
| needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, | |||||
| 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, | |||||
| 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, | |||||
| 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, | |||||
| 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, | |||||
| 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, | |||||
| 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, | |||||
| 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, | |||||
| 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, | |||||
| 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, | |||||
| 'xh': None, 'yi': None, 'yo': None, 'zu': None} | |||||
| jobs.put(("discovery", desit.split(":", 1)[1], None)) | |||||
| elif desit.split(":", 1)[0] == "channel": | |||||
| jobs.put(("channel", None, desit.split(":", 1)[1])) | |||||
| elif desit.split(":", 1)[0] == "playlist": | |||||
| jobs.put(("playlist", None, desit.split(":", 1)[1])) | |||||
| elif desit.split(":", 1)[0] == "mixplaylist": | |||||
| jobs.put(("mixplaylist", None, desit.split(":", 1)[1])) | |||||
| else: | |||||
| print("Ignoring item for now", desit) | |||||
| desit = tracker.request_item_from_tracker() | |||||
| print("New task:", desit) | |||||
| if desit: | |||||
| if desit.split(":", 1)[0] == "video": | |||||
| needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, | |||||
| 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, | |||||
| 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, | |||||
| 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, | |||||
| 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, | |||||
| 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, | |||||
| 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, | |||||
| 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, | |||||
| 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, | |||||
| 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, | |||||
| 'xh': None, 'yi': None, 'yo': None, 'zu': None} | |||||
| needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, | |||||
| 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, | |||||
| 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, | |||||
| 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, | |||||
| 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, | |||||
| 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, | |||||
| 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, | |||||
| 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, | |||||
| 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, | |||||
| 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, | |||||
| 'xh': None, 'yi': None, 'yo': None, 'zu': None} | |||||
| jobs.put(("discovery", desit.split(":", 1)[1], None)) | |||||
| elif desit.split(":", 1)[0] == "channel": | |||||
| jobs.put(("channel", None, desit.split(":", 1)[1])) | |||||
| elif desit.split(":", 1)[0] == "playlist": | |||||
| jobs.put(("playlist", None, desit.split(":", 1)[1])) | |||||
| elif desit.split(":", 1)[0] == "mixplaylist": | |||||
| jobs.put(("mixplaylist", None, desit.split(":", 1)[1])) | |||||
| else: | else: | ||||
| print("Ignoring item for now", desit) | print("Ignoring item for now", desit) | ||||
| else: | else: | ||||
| break | |||||
| print("Ignoring item for now", desit) | |||||
| else: | else: | ||||
| break | break | ||||
| @@ -338,8 +302,4 @@ for x in threads: | |||||
| threads.remove(x) | threads.remove(x) | ||||
| del x | del x | ||||
| if not shouldgetjob: | |||||
| print("PROTECTION MECHANISM #3 WAS SOMEHOW TRIGERRED") | |||||
| print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.") | |||||
| print("Exiting...") | |||||
| print("Exiting...") | |||||