diff --git a/worker.py b/worker.py index 8fad9d2..f140f9d 100644 --- a/worker.py +++ b/worker.py @@ -51,9 +51,6 @@ def submitfunc(submitqueue): itype, ival = submitqueue.get() tracker.add_item_to_tracker(itype, ival) -WORKER_VERSION = 1 -SERVER_BASE_URL = "http://localhost:5000" - langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', @@ -158,6 +155,11 @@ while not gkiller.kill_now: except: pass + try: + mkdir("directory") + except: + pass + batchcontent.clear() # Get a batch ID @@ -277,7 +279,7 @@ while not gkiller.kill_now: for fol in listdir("out"): if isdir("out/"+fol): - make_archive("out/"+fol, "zip", "out/"+fol) #check this + make_archive("directory/"+fol, "zip", "out/"+fol) #check this targetloc = None while not targetloc: @@ -288,15 +290,7 @@ while not gkiller.kill_now: print("Waiting 5 minutes...") sleep(300) - for zipf in listdir("out"): - if isfile(zipf) in zipf.endswith(".zip"): - if targetloc.startswith("rsync"): - system("rsync out/"+zipf+" "+targetloc) - elif targetloc.startswith("http"): - upzipf = open("out/"+zipf, "rb") - requests.post(targetloc, data=upzipf) - upzipf.close() - #upload it! + system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 --files-from=- /directory/ "+targetloc) # Report the batch as complete for itemb in batchcontent: