diff --git a/libgrabsite/wpull_tweaks.py b/libgrabsite/wpull_tweaks.py index 6bef92d..401b3d9 100644 --- a/libgrabsite/wpull_tweaks.py +++ b/libgrabsite/wpull_tweaks.py @@ -4,6 +4,7 @@ import functools from wpull.database.sqltable import SQLiteURLTable from wpull.document.html import HTMLReader +from wpull.network.dns import Resolver from wpull.processor.rule import ProcessingRule from libgrabsite import dupespotter @@ -55,8 +56,15 @@ class DupeSpottingProcessingRule(ProcessingRule): super().scrape_document(item_session) +class NoDnspythonResolver(Resolver): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dns_python_enabled = False + + def activate(app_session): app_session.factory.class_map['URLTableImplementation'] = NoFsyncSQLTable + app_session.factory.class_map['Resolver'] = NoDnspythonResolver if int(os.environ["DUPESPOTTER_ENABLED"]): dupes_db_location = os.path.join(os.environ["GRAB_SITE_WORKING_DIR"], "dupes_db")