From cf7bfa726c41c47cd3eca751dd56ecd16d15cc82 Mon Sep 17 00:00:00 2001
From: tech234a <46801700+tech234a@users.noreply.github.com>
Date: Mon, 21 Sep 2020 00:27:47 -0400
Subject: [PATCH] Take out the trash

---
 export.py | 3 +++
 worker.py | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/export.py b/export.py
index 612c90d..cb764a7 100644
--- a/export.py
+++ b/export.py
@@ -29,6 +29,8 @@ from datetime import timedelta
 
 from json import dumps
 
+from gc import collect
+
 import requests
 
 # https://docs.python.org/3/library/html.parser.html
@@ -71,6 +73,7 @@ class MyHTMLParser(HTMLParser):
 
 def subprrun(jobs, headers):
     while not jobs.empty():
+        collect() #cleanup memory
         langcode, vid = jobs.get()
         vid = vid.strip()
         print(langcode, vid)
diff --git a/worker.py b/worker.py
index 0990aab..a037b5b 100644
--- a/worker.py
+++ b/worker.py
@@ -9,6 +9,8 @@ from shutil import make_archive, rmtree
 
 from queue import Queue
 
+from gc import collect
+
 from discovery import getmetadata
 from export import subprrun
 
@@ -99,6 +101,8 @@ while True:
         sleep(600)
 
 while True:
+    collect() #cleanup
+
     try:
         mkdir("out")
     except: