|
- diff --git a/libgrabsite/wpull_tweaks.py b/libgrabsite/wpull_tweaks.py
- index 6bef92d..401b3d9 100644
- --- a/libgrabsite/wpull_tweaks.py
- +++ b/libgrabsite/wpull_tweaks.py
- @@ -4,6 +4,7 @@ import functools
-
- from wpull.database.sqltable import SQLiteURLTable
- from wpull.document.html import HTMLReader
- +from wpull.network.dns import Resolver
- from wpull.processor.rule import ProcessingRule
-
- from libgrabsite import dupespotter
- @@ -55,8 +56,15 @@ class DupeSpottingProcessingRule(ProcessingRule):
- super().scrape_document(item_session)
-
-
- +class NoDnspythonResolver(Resolver):
- + def __init__(self, *args, **kwargs):
- + super().__init__(*args, **kwargs)
- + self.dns_python_enabled = False
- +
- +
- def activate(app_session):
- app_session.factory.class_map['URLTableImplementation'] = NoFsyncSQLTable
- + app_session.factory.class_map['Resolver'] = NoDnspythonResolver
-
- if int(os.environ["DUPESPOTTER_ENABLED"]):
- dupes_db_location = os.path.join(os.environ["GRAB_SITE_WORKING_DIR"], "dupes_db")
|
