Bläddra i källkod

Make all `*_temporary_metadata` methods take/return unique names rather than filenames

tags/v1.1
JustAnotherArchivist 1 år sedan
förälder
incheckning
cc802f7727
1 ändrade filer med 8 tillägg och 5 borttagningar
  1. +8
    -5
      codearchiver/storage.py

+ 8
- 5
codearchiver/storage.py Visa fil

@@ -83,7 +83,7 @@ class Storage(abc.ABC):


@abc.abstractmethod @abc.abstractmethod
def search_temporary_metadata(self, criteria: list[tuple[str, typing.Union[str, tuple[str]]]]) -> collections.abc.Iterator[str]: def search_temporary_metadata(self, criteria: list[tuple[str, typing.Union[str, tuple[str]]]]) -> collections.abc.Iterator[str]:
'''Same as `search_metadata`, but for the temporary metadata written by `add_temporary_metadata`.'''
'''Same as `search_metadata`, but for the temporary metadata written by `add_temporary_metadata`, returning the unique names instead.'''


@abc.abstractmethod @abc.abstractmethod
def open_temporary_metadata(self, name: str) -> typing.TextIO: def open_temporary_metadata(self, name: str) -> typing.TextIO:
@@ -190,13 +190,14 @@ class DirectoryStorage(Storage):
# Build a filename based on the current time in nanoseconds and a (truncated) hash of the metadata; this should guaranteed uniqueness to a sufficient degree. # Build a filename based on the current time in nanoseconds and a (truncated) hash of the metadata; this should guaranteed uniqueness to a sufficient degree.
serialised = metadata.serialise().encode('utf-8') serialised = metadata.serialise().encode('utf-8')
metadataHash = hashlib.sha512(serialised).hexdigest()[:128] metadataHash = hashlib.sha512(serialised).hexdigest()[:128]
filename = f'tmp_{time.time_ns()}_{metadataHash}_codearchiver_temporary_metadata.txt'
name = f'tmp_{time.time_ns()}_{metadataHash}'
filename = f'{name}_codearchiver_temporary_metadata.txt'
self._ensure_directory() self._ensure_directory()
_logger.info(f'Writing temporary metadata to {filename}') _logger.info(f'Writing temporary metadata to {filename}')
with open(os.path.join(self._directory, filename), 'xb') as fp: with open(os.path.join(self._directory, filename), 'xb') as fp:
fp.write(serialised) fp.write(serialised)
_logger.info('Done writing temporary metadata file') _logger.info('Done writing temporary metadata file')
return filename
return name


def search_temporary_metadata(self, criteria): def search_temporary_metadata(self, criteria):
yield from self.search_metadata(criteria, _suffix = '_codearchiver_temporary_metadata.txt') yield from self.search_metadata(criteria, _suffix = '_codearchiver_temporary_metadata.txt')
@@ -207,14 +208,15 @@ class DirectoryStorage(Storage):
yield fp yield fp


def replace_temporary_metadata(self, name, filename, metadata): def replace_temporary_metadata(self, name, filename, metadata):
_logger.info(f'Replacing temporary metadata file {name}')
self.put(filename, metadata) self.put(filename, metadata)
self.remove_temporary_metadata(name) self.remove_temporary_metadata(name)


def remove_temporary_metadata(self, name): def remove_temporary_metadata(self, name):
if not name.endswith('_codearchiver_temporary_metadata.txt'):
if name.endswith('_codearchiver_temporary_metadata.txt'):
raise RuntimeError('invalid temporary metadata name provided') raise RuntimeError('invalid temporary metadata name provided')
_logger.info(f'Removing temporary metadata file {name}') _logger.info(f'Removing temporary metadata file {name}')
os.remove(os.path.join(self._directory, name))
os.remove(os.path.join(self._directory, f'{name}_codearchiver_temporary_metadata.txt'))


def wait_temporary_metadata(self, names, sleepTime = 5): def wait_temporary_metadata(self, names, sleepTime = 5):
_logger.info(f'Waiting for temporary metadata: {names!r}') _logger.info(f'Waiting for temporary metadata: {names!r}')
@@ -223,6 +225,7 @@ class DirectoryStorage(Storage):
with self.lock(blocking = False) as locked: with self.lock(blocking = False) as locked:
if locked: if locked:
remaining = {filename for filename in remaining if os.path.exists(filename)} remaining = {filename for filename in remaining if os.path.exists(filename)}
_logger.debug(f'Remaining: {remaining!r}')
if not remaining: if not remaining:
break break
time.sleep(sleepTime) time.sleep(sleepTime)


Laddar…
Avbryt
Spara