diff --git a/export.py b/export.py index 612c90d..cb764a7 100644 --- a/export.py +++ b/export.py @@ -29,6 +29,8 @@ from datetime import timedelta from json import dumps +from gc import collect + import requests # https://docs.python.org/3/library/html.parser.html @@ -71,6 +73,7 @@ class MyHTMLParser(HTMLParser): def subprrun(jobs, headers): while not jobs.empty(): + collect() #cleanup memory langcode, vid = jobs.get() vid = vid.strip() print(langcode, vid) diff --git a/worker.py b/worker.py index 0990aab..a037b5b 100644 --- a/worker.py +++ b/worker.py @@ -9,6 +9,8 @@ from shutil import make_archive, rmtree from queue import Queue +from gc import collect + from discovery import getmetadata from export import subprrun @@ -99,6 +101,8 @@ while True: sleep(600) while True: + collect() #cleanup + try: mkdir("out") except: