Take out the trash

export.py

@@ -29,6 +29,8 @@ from datetime import timedelta
 
 from json import dumps
 
+from gc import collect
+
 import requests
 
 # https://docs.python.org/3/library/html.parser.html
@@ -71,6 +73,7 @@ class MyHTMLParser(HTMLParser):
 
 def subprrun(jobs, headers):
     while not jobs.empty():
+        collect() #cleanup memory
         langcode, vid = jobs.get()
         vid = vid.strip()
         print(langcode, vid)

worker.py

@@ -9,6 +9,8 @@ from shutil import make_archive, rmtree
 
 from queue import Queue
 
+from gc import collect
+
 from discovery import getmetadata
 from export import subprrun
 
@@ -99,6 +101,8 @@ while True:
         sleep(600)
 
 while True:
+    collect() #cleanup
+
     try:
         mkdir("out")
     except: