@@ -3,10 +3,12 @@ import contextlib | |||||
import datetime | import datetime | ||||
import inspect | import inspect | ||||
import logging | import logging | ||||
import os | |||||
import requests.models | import requests.models | ||||
# Imported in parse_args() and main() after setting up the logger: | # Imported in parse_args() and main() after setting up the logger: | ||||
#import codearchiver.core | #import codearchiver.core | ||||
#import codearchiver.modules | #import codearchiver.modules | ||||
#import codearchiver.storage | |||||
#import codearchiver.version | #import codearchiver.version | ||||
import tempfile | import tempfile | ||||
@@ -182,13 +184,24 @@ def main(): | |||||
setup_logging() | setup_logging() | ||||
args = parse_args() | args = parse_args() | ||||
configure_logging(args.verbosity, args.dumpLocals) | configure_logging(args.verbosity, args.dumpLocals) | ||||
_logger = logging.getLogger(__name__) | |||||
import codearchiver.core | import codearchiver.core | ||||
import codearchiver.modules | import codearchiver.modules | ||||
import codearchiver.storage | |||||
with _dump_locals_on_exception(): | with _dump_locals_on_exception(): | ||||
inputUrl = codearchiver.core.InputURL(args.url) | inputUrl = codearchiver.core.InputURL(args.url) | ||||
module = codearchiver.core.get_module_instance(inputUrl) | module = codearchiver.core.get_module_instance(inputUrl) | ||||
module.process() | |||||
storage = codearchiver.storage.DirectoryStorage(os.getcwd()) | |||||
with tempfile.TemporaryDirectory(prefix = 'tmp.codearchiver.', dir = os.getcwd()) as td: | |||||
_logger.debug(f'Running in {td}') | |||||
os.chdir(td) | |||||
try: | |||||
result = module.process() | |||||
storage.queue_result(result) | |||||
storage.put_queue() | |||||
finally: | |||||
os.chdir('..') | |||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
main() | main() |
@@ -0,0 +1,66 @@ | |||||
import abc | |||||
import codearchiver.core | |||||
import contextlib | |||||
import logging | |||||
import os.path | |||||
import shutil | |||||
import typing | |||||
_logger = logging.getLogger(__name__) | |||||
class Storage(abc.ABC): | |||||
def __init__(self): | |||||
self._queue = [] | |||||
def queue(self, filename: str): | |||||
'''Queue a local file for putting into storage. Note that nothing is written to storage until `put_queue` is called.''' | |||||
_logger.debug(f'Queueing {filename}') | |||||
self._queue.append(filename) | |||||
def queue_result(self, result: codearchiver.core.Result): | |||||
for fn in result.files: | |||||
self.queue(fn) | |||||
for _, subresult in result.submoduleResults: | |||||
self.queue_result(subresult) | |||||
@property | |||||
def queued_files(self) -> typing.List[str]: | |||||
return self._queue[:] | |||||
@abc.abstractmethod | |||||
def put_queue(self): | |||||
'''Put all queued files into storage. If an error occurs, partial copies may remain in storage. If it completes, the local input copy is removed.''' | |||||
@abc.abstractmethod | |||||
@contextlib.contextmanager | |||||
def open(self, filename: str) -> typing.Iterator[typing.BinaryIO]: | |||||
'''Open a file from storage.''' | |||||
class DirectoryStorage(Storage): | |||||
def __init__(self, directory): | |||||
super().__init__() | |||||
self._directory = directory | |||||
def _check_directory(self): | |||||
exists = os.path.exists(self._directory) | |||||
if exists and not os.path.isdir(self._directory): | |||||
raise NotADirectoryError(self._directory) | |||||
return exists | |||||
def _ensure_directory(self): | |||||
if not self._check_directory(): | |||||
os.makedirs(self._directory) | |||||
def put_queue(self): | |||||
self._ensure_directory() | |||||
for fn in self.queued_files: | |||||
_logger.info(f'Moving {fn} to {self._directory}') | |||||
shutil.move(fn, self._directory) | |||||
@contextlib.contextmanager | |||||
def open(self, filename): | |||||
with open(filename, 'rb') as fp: | |||||
yield fp |